Add support for user defined package sources API

This lives under /api/v0/projects/source/*

See the documentation for details

docs/lorax-composer.rst

@@ -102,3 +102,55 @@ the results directory, or it could do some post-processing on it. The end of
 the function should always clean up the ``./compose/`` directory, removing any
 unneeded extra files. This is especially true for the ``live-iso`` since it produces
 the contents of the iso as well as the boot.iso itself.
+
+Package Sources
+---------------
+
+By default lorax-composer uses the host's configured repositories. It copies
+the ``*.repo`` files from ``/etc/yum.repos.d/`` into
+``/var/lib/lorax/composer/repos.d/`` at startup, these are immutable system
+repositories and cannot be deleted or changed. If you want to add additional
+repos you can put them into ``/var/lib/lorax/composer/repos.d/`` or use the
+``/api/v0/projects/source/*`` API routes to create them.
+
+The new source can be added by doing a POST to the ``/api/v0/projects/source/new``
+route using JSON (with `Content-Type` header set to `application/json`) or TOML
+(with it set to `text/x-toml`).  The format of the source looks like this (in
+TOML)::
+
+    name = "custom-source-1"
+    url = "https://url/path/to/repository/"
+    type = "yum-baseurl"
+    proxy = "https://proxy-url/"
+    check_ssl = true
+    check_gpg = true
+    gpgkey_urls = ["https://url/path/to/gpg-key"]
+
+The ``proxy`` and ``gpgkey_urls`` entries are optional. All of the others are required. The supported
+types for the urls are:
+
+* ``yum-baseurl`` is a URL to a yum repository.
+* ``yum-mirrorlist`` is a URL for a mirrorlist.
+* ``yum-metalink`` is a URL for a metalink.
+
+If ``check_ssl`` is true the https certificates must be valid. If they are self-signed you can either set
+this to false, or add your Certificate Authority to the host system.
+
+If ``check_gpg`` is true the GPG key must either be installed on the host system, or ``gpgkey_urls``
+should point to it.
+
+You can edit an existing source (other than system sources), by doing a POST to the ``new`` route
+with the new version of the source. It will overwrite the previous one.
+
+A list of existing sources is available from ``/api/v0/projects/source/list``, and detailed info
+on a source can be retrieved with the ``/api/v0/projects/source/info/<source-name>`` route. By default
+it returns JSON but it can also return TOML if ``?format=toml`` is added to the request.
+
+Non-system sources can be deleted by doing a ``DELETE`` request to the
+``/api/v0/projects/source/delete/<source-name>`` route.
+
+The documentation for the source API routes can be `found here <pylorax.api.html#api-v0-projects-source-list>`_
+
+The configured sources are used for all blueprint depsolve operations, and for composing images.
+When adding additional sources you must make sure that the packages in the source do not
+conflict with any other package sources, otherwise depsolving will fail.

src/pylorax/api/config.py

@@ -44,9 +44,9 @@ def configure(conf_file="/etc/lorax/composer.conf", root_dir="/", test_config=Fa
     conf.add_section("composer")
     conf.set("composer", "share_dir", os.path.realpath(joinpaths(root_dir, "/usr/share/lorax/")))
     conf.set("composer", "lib_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/composer/")))
+    conf.set("composer", "repo_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/composer/repos.d/")))
     conf.set("composer", "yum_conf", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/yum.conf")))
     conf.set("composer", "yum_root", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/yum/root/")))
-    conf.set("composer", "repo_dir", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/repos.d/")))
     conf.set("composer", "cache_dir", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/cache/")))
     conf.set("composer", "tmp", os.path.realpath(joinpaths(root_dir, "/var/tmp/")))
 

src/pylorax/api/projects.py

@@ -17,6 +17,10 @@
 import logging
 log = logging.getLogger("lorax-composer")
 
+import os
+from ConfigParser import ConfigParser
+import yum
+from glob import glob
 import time
 
 from yum.Errors import YumBaseError
@@ -315,3 +319,175 @@ def modules_info(yb, module_names):
         module["dependencies"] = projects_depsolve(yb, [(module["name"], "*")])
 
     return modules
+
+def repo_to_source(repo, system_source):
+    """Return a Weldr Source dict created from the YumRepository
+
+    :param repo: Yum Repository
+    :type repo: yum.yumRepo.YumRepository
+    :param system_source: True if this source is an immutable system source
+    :type system_source: bool
+    :returns: A dict with Weldr Source fields filled in
+    :rtype: dict
+
+    Example::
+
+        {
+          "check_gpg": true,
+          "check_ssl": true,
+          "gpgkey_url": [
+            "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64"
+          ],
+          "name": "fedora",
+          "proxy": "http://proxy.brianlane.com:8123",
+          "system": true
+          "type": "yum-metalink",
+          "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64"
+        }
+
+    """
+    source = {"name": repo.id, "system": system_source}
+    if repo.baseurl:
+        source["url"] = repo.baseurl[0]
+        source["type"] = "yum-baseurl"
+    elif repo.metalink:
+        source["url"] = repo.metalink
+        source["type"] = "yum-metalink"
+    elif repo.mirrorlist:
+        source["url"] = repo.mirrorlist
+        source["type"] = "yum-mirrorlist"
+    else:
+        raise RuntimeError("Repo has no baseurl, metalink, or mirrorlist")
+
+    # proxy is optional
+    if repo.proxy:
+        source["proxy"] = repo.proxy
+
+    if not repo.sslverify:
+        source["check_ssl"] = False
+    else:
+        source["check_ssl"] = True
+
+    if not repo.gpgcheck:
+        source["check_gpg"] = False
+    else:
+        source["check_gpg"] = True
+
+    if repo.gpgkey:
+        source["gpgkey_urls"] = repo.gpgkey
+
+    return source
+
+def source_to_repo(source):
+    """Return a yum YumRepository object created from a source dict
+
+    :param source: A Weldr source dict
+    :type source: dict
+    :returns: A yum YumRepository object
+    :rtype: yum.yumRepo.YumRepository
+
+    Example::
+
+        {
+          "check_gpg": True,
+          "check_ssl": True,
+          "gpgkey_urls": [
+            "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64"
+          ],
+          "name": "fedora",
+          "proxy": "http://proxy.brianlane.com:8123",
+          "system": True
+          "type": "yum-metalink",
+          "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64"
+        }
+
+    """
+    repo = yum.yumRepo.YumRepository(source["name"])
+    if source["type"] == "yum-baseurl":
+        repo.baseurl = [source["url"]]
+    elif source["type"] == "yum-metalink":
+        repo.metalink = source["url"]
+    elif source["type"] == "yum-mirrorlist":
+        repo.mirrorlist = source["url"]
+
+    if "proxy" in source:
+        repo.proxy = source["proxy"]
+
+    if source["check_ssl"]:
+        repo.sslverify = True
+    else:
+        repo.sslverify = False
+
+    if source["check_gpg"]:
+        repo.gpgcheck = True
+    else:
+        repo.gpgcheck = False
+
+    if "gpgkey_urls" in source:
+        repo.gpgkey = source["gpgkey_urls"]
+
+    repo.enable()
+
+    return repo
+
+def get_source_ids(source_path):
+    """Return a list of the source ids in a file
+
+    :param source_path: Full path and filename of the source (yum repo) file
+    :type source_path: str
+    :returns: A list of source id strings
+    :rtype: list of str
+    """
+    if not os.path.exists(source_path):
+        return []
+
+    cfg = ConfigParser()
+    cfg.read(source_path)
+    return cfg.sections()
+
+def get_repo_sources(source_glob):
+    """Return a list of sources from a directory of yum repositories
+
+    :param source_glob: A glob to use to match the source files, including full path
+    :type source_glob: str
+    :returns: A list of the source ids in all of the matching files
+    :rtype: list of str
+    """
+    sources = []
+    for f in glob(source_glob):
+        sources.extend(get_source_ids(f))
+    return sources
+
+def delete_repo_source(source_glob, source_name):
+    """Delete a source from a repo file
+
+    :param source_glob: A glob of the repo sources to search
+    :type source_glob: str
+    :returns: None
+    :raises: ProjectsError if there was a problem
+
+    A repo file may have multiple sources in it, delete only the selected source.
+    If it is the last one in the file, delete the file.
+
+    WARNING: This will delete ANY source, the caller needs to ensure that a system
+    source_name isn't passed to it.
+    """
+    found = False
+    for f in glob(source_glob):
+        try:
+            cfg = ConfigParser()
+            cfg.read(f)
+            if source_name in cfg.sections():
+                found = True
+                cfg.remove_section(source_name)
+                # If there are other sections, rewrite the file without the deleted one
+                if len(cfg.sections()) > 0:
+                    with open(f, "w") as cfg_file:
+                        cfg.write(cfg_file)
+                else:
+                    # No sections left, just delete the file
+                    os.unlink(f)
+        except Exception as e:
+            raise ProjectsError("Problem deleting repo source %s: %s" % (source_name, str(e)))
+    if not found:
+        raise ProjectsError("source %s not found" % source_name)

src/pylorax/api/queue.py

@@ -89,7 +89,7 @@ def monitor(cfg):
         # Pick the oldest and move it into ./run/
         if not uuids:
             # No composes left to process, sleep for a bit
-            time.sleep(30)
+            time.sleep(5)
         else:
             src = joinpaths(cfg.composer_dir, "queue/new", uuids[0])
             dst = joinpaths(cfg.composer_dir, "queue/run", uuids[0])
@@ -192,7 +192,7 @@ def make_compose(cfg, results_dir):
         test_path = joinpaths(results_dir, "TEST")
         if os.path.exists(test_path):
             # Pretend to run the compose
-            time.sleep(10)
+            time.sleep(5)
             try:
                 test_mode = int(open(test_path, "r").read())
             except Exception:

src/pylorax/api/v0.py

@@ -501,6 +501,94 @@ POST `/api/v0/blueprints/tag/<blueprint_name>`
         ]
       }
 
+`/api/v0/projects/source/list`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  Return the list of repositories used for depsolving and installing packages.
+
+  Example::
+
+      {
+        "sources": [
+          "fedora",
+          "fedora-cisco-openh264",
+          "fedora-updates-testing",
+          "fedora-updates"
+        ]
+      }
+
+`/api/v0/projects/source/info/<source-names>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  Return information about the comma-separated list of source names. Or all of the
+  sources if '*' is passed. Note that general globbing is not supported, only '*'.
+
+  immutable system sources will have the "system" field set to true. User added sources
+  will have it set to false. System sources cannot be changed or deleted.
+
+  Example::
+
+      {
+        "errors": [],
+        "sources": {
+          "fedora": {
+            "check_gpg": true,
+            "check_ssl": true,
+            "gpgkey_urls": [
+              "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64"
+            ],
+            "name": "fedora",
+            "proxy": "http://proxy.brianlane.com:8123",
+            "system": true,
+            "type": "yum-metalink",
+            "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64"
+          }
+        }
+      }
+
+POST `/api/v0/projects/source/new`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  Add (or change) a source for use when depsolving blueprints and composing images.
+
+  The ``proxy`` and ``gpgkey_urls`` entries are optional. All of the others are required. The supported
+  types for the urls are:
+
+  * ``yum-baseurl`` is a URL to a yum repository.
+  * ``yum-mirrorlist`` is a URL for a mirrorlist.
+  * ``yum-metalink`` is a URL for a metalink.
+
+  If ``check_ssl`` is true the https certificates must be valid. If they are self-signed you can either set
+  this to false, or add your Certificate Authority to the host system.
+
+  If ``check_gpg`` is true the GPG key must either be installed on the host system, or ``gpgkey_urls``
+  should point to it.
+
+  You can edit an existing source (other than system sources), by doing a POST
+  of the new version of the source. It will overwrite the previous one.
+
+  Example::
+
+      {
+          "name": "custom-source-1",
+          "url": "https://url/path/to/repository/",
+          "type": "yum-baseurl",
+          "check_ssl": true,
+          "check_gpg": true,
+          "gpgkey_urls": [
+              "https://url/path/to/gpg-key"
+          ]
+      }
+
+DELETE `/api/v0/projects/source/delete/<source-name>`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+  Delete a user added source. This will fail if a system source is passed to
+  it.
+
+  The response will be a status response with `status` set to true, or an
+  error response with it set to false and an error message included.
+
 `/api/v0/modules/list[?offset=0&limit=20]`
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -878,17 +966,21 @@ log = logging.getLogger("lorax-composer")
 
 import os
 from flask import jsonify, request, Response, send_file
+import pytoml as toml
 
+from pylorax.sysutils import joinpaths
 from pylorax.api.compose import start_build, compose_types
 from pylorax.api.crossdomain import crossdomain
 from pylorax.api.projects import projects_list, projects_info, projects_depsolve
-from pylorax.api.projects import modules_list, modules_info, ProjectsError
+from pylorax.api.projects import modules_list, modules_info, ProjectsError, repo_to_source
+from pylorax.api.projects import get_repo_sources, delete_repo_source, source_to_repo
 from pylorax.api.queue import queue_status, build_status, uuid_delete, uuid_status, uuid_info
 from pylorax.api.queue import uuid_tar, uuid_image, uuid_cancel, uuid_log
 from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_filename, list_commits
 from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe
 from pylorax.api.recipes import tag_recipe_commit, recipe_diff
 from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete
+from pylorax.api.yumbase import update_metadata
 
 # The API functions don't actually get called by any code here
 # pylint: disable=unused-variable
@@ -1300,6 +1392,121 @@ def v0_api(api):
 
         return jsonify(projects=deps)
 
+    @api.route("/api/v0/projects/source/list")
+    @crossdomain(origin="*")
+    def v0_projects_source_list():
+        """Return the list of source names"""
+        with api.config["YUMLOCK"].lock:
+            repos = list(api.config["YUMLOCK"].yb.repos.listEnabled())
+        sources = sorted([r.id for r in repos])
+        return jsonify(sources=sources)
+
+    @api.route("/api/v0/projects/source/info/<source_names>")
+    @crossdomain(origin="*")
+    def v0_projects_source_info(source_names):
+        """Return detailed info about the list of sources"""
+        out_fmt = request.args.get("format", "json")
+
+        # Return info on all of the sources
+        if source_names == "*":
+            with api.config["YUMLOCK"].lock:
+                source_names = ",".join(r.id for r in api.config["YUMLOCK"].yb.repos.listEnabled())
+
+        sources = {}
+        errors = []
+        system_sources = get_repo_sources("/etc/yum.repos.d/*.repo")
+        for source in source_names.split(","):
+            with api.config["YUMLOCK"].lock:
+                repo = api.config["YUMLOCK"].yb.repos.repos.get(source, None)
+            if not repo:
+                errors.append("%s is not a valid source" % source)
+                continue
+            sources[repo.id] = repo_to_source(repo, repo.id in system_sources)
+
+        if out_fmt == "toml":
+            # With TOML output we just want to dump the raw sources, skipping the errors
+            return toml.dumps(sources)
+        else:
+            return jsonify(sources=sources, errors=errors)
+
+    @api.route("/api/v0/projects/source/new", methods=["POST"])
+    @crossdomain(origin="*")
+    def v0_projects_source_new():
+        """Add a new package source. Or change an existing one"""
+        if request.headers['Content-Type'] == "text/x-toml":
+            source = toml.loads(request.data)
+        else:
+            source = request.get_json(cache=False)
+
+        system_sources = get_repo_sources("/etc/yum.repos.d/*.repo")
+        if source["name"] in system_sources:
+            return jsonify(status=False, errors=["%s is a system source, it cannot be deleted." % source["name"]]), 400
+
+        try:
+            # Delete it from yum (if it exists) and replace it with the new one
+            with api.config["YUMLOCK"].lock:
+                yb = api.config["YUMLOCK"].yb
+                # If this repo already exists, delete it and replace it with the new one
+                repos = list(r.id for r in yb.repos.listEnabled())
+                if source["name"] in repos:
+                    yb.repos.delete(source["name"])
+
+                # XXX - BCL DIAGNOSTIC
+                repos = list(r.id for r in yb.repos.listEnabled())
+                if source["name"] in repos:
+                    return jsonify(status=False, errors=["Failed to delete Yum repo %s" % source["name"]]), 400
+
+                repo = source_to_repo(source)
+                yb.repos.add(repo)
+
+                log.info("Updating repository metadata after adding %s", source["name"])
+                update_metadata(yb)
+
+            # Write the new repo to disk, replacing any existing ones
+            repo_dir = api.config["COMPOSER_CFG"].get("composer", "repo_dir")
+
+            # Remove any previous sources with this name, ignore it if it isn't found
+            try:
+                delete_repo_source(joinpaths(repo_dir, "*.repo"), source["name"])
+            except ProjectsError:
+                pass
+
+            # Make sure the source name can't contain a path traversal by taking the basename
+            source_path = joinpaths(repo_dir, os.path.basename("%s.repo" % source["name"]))
+            with open(source_path, "w") as f:
+                f.write(str(repo))
+        except Exception as e:
+            return jsonify(status=False, errors=[str(e)]), 400
+
+        return jsonify(status=True)
+
+    @api.route("/api/v0/projects/source/delete/<source_name>", methods=["DELETE"])
+    @crossdomain(origin="*")
+    def v0_projects_source_delete(source_name):
+        """Delete the named source and return a status response"""
+        system_sources = get_repo_sources("/etc/yum.repos.d/*.repo")
+        if source_name in system_sources:
+            return jsonify(status=False, errors=["%s is a system source, it cannot be deleted." % source_name]), 400
+        share_dir = api.config["COMPOSER_CFG"].get("composer", "repo_dir")
+        try:
+            # Remove the file entry for the source
+            delete_repo_source(joinpaths(share_dir, "*.repo"), source_name)
+
+            # Delete the repo
+            with api.config["YUMLOCK"].lock:
+                yb = api.config["YUMLOCK"].yb
+                repos = list(r.id for r in yb.repos.listEnabled())
+                if source_name in repos:
+                    yb.repos.delete(source_name)
+                    log.info("Updating repository metadata after removing %s", source_name)
+                    update_metadata(yb)
+
+        except ProjectsError as e:
+            log.error("(v0_projects_source_delete) %s", str(e))
+            return jsonify(status=False, errors=[str(e)]), 400
+
+        return jsonify(status=True)
+
     @api.route("/api/v0/modules/list")
     @api.route("/api/v0/modules/list/<module_names>")
     @crossdomain(origin="*")

src/pylorax/api/yumbase.py

@@ -103,11 +103,21 @@ def get_base_object(conf):
 
     # Update the metadata from the enabled repos to speed up later operations
     log.info("Updating yum repository metadata")
+    update_metadata(yb)
+
+    return yb
+
+def update_metadata(yb):
+    """Update the metadata for all the enabled repos
+
+    :param yb: The Yum base object
+    :type yb: yum.YumBase
+    :returns: None
+    :rtype: None
+    """
     for r in yb.repos.sort():
         r.metadata_expire = 0
         r.mdpolicy = "group:all"
     yb.doRepoSetup()
     yb.repos.doSetup()
     yb.repos.populateSack(mdtype='all', cacheonly=1)
-
-    return yb

tests/pylorax/repos/multiple.repo

@@ -0,0 +1,47 @@
+[lorax-1]
+name=Lorax test repo 1
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False
+
+[lorax-2]
+name=Lorax test repo 2
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False
+
+[lorax-3]
+name=Lorax test repo 3
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False
+
+[lorax-4]
+name=Lorax test repo 4
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False

tests/pylorax/repos/other.repo

@@ -0,0 +1,11 @@
+[other-repo]
+name=Other repo
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False

tests/pylorax/repos/single.repo

@@ -0,0 +1,11 @@
+[single-repo]
+name=One repo in the file
+failovermethod=priority
+baseurl=file:///tmp/lorax-empty-repo/
+enabled=1
+metadata_expire=7d
+repo_gpgcheck=0
+type=rpm
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch
+skip_if_unavailable=False

tests/pylorax/source/replace-repo.toml

@@ -0,0 +1,6 @@
+name = "single-repo"
+url = "file:///tmp/lorax-empty-repo/"
+type = "yum-baseurl"
+check_ssl = false
+check_gpg = true
+gpgkey_urls = []

tests/pylorax/source/test-repo.json

@@ -0,0 +1 @@
+{"name": "new-repo-1", "url": "file:///tmp/lorax-empty-repo/", "type": "yum-baseurl", "check_ssl": true, "check_gpg": true, "gpgkey_urls": ["file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch"]}

tests/pylorax/source/test-repo.toml

@@ -0,0 +1,6 @@
+name = "new-repo-2"
+url = "file:///tmp/lorax-empty-repo/"
+type = "yum-baseurl"
+check_ssl = true
+check_gpg = true
+gpgkey_urls = ["file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch"]

tests/pylorax/test_projects.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+from glob import glob
 import os
 import mock
 import time
@@ -23,11 +24,13 @@ import unittest
 
 from yum.Errors import YumBaseError
 
+from pylorax.sysutils import joinpaths
 from pylorax.api.config import configure, make_yum_dirs
 from pylorax.api.projects import api_time, api_changelog, yaps_to_project, yaps_to_project_info
 from pylorax.api.projects import tm_to_dep, yaps_to_module, projects_list, projects_info, projects_depsolve
 from pylorax.api.projects import modules_list, modules_info, ProjectsError, dep_evra, dep_nevra
 from pylorax.api.yumbase import get_base_object
+from pylorax.api.projects import repo_to_source, get_repo_sources, delete_repo_source, source_to_repo
 
 
 class Yaps(object):
@@ -247,3 +250,215 @@ class ConfigureTest(unittest.TestCase):
     def test_configure_reads_non_existing_file(self):
         config = configure(conf_file=self.conf_file + '.non-existing')
         self.assertEqual(config.get('composer', 'cache_dir'), '/var/tmp/composer/cache')
+
+class FakeRepoBaseUrl(object):
+    id = "fake-repo-baseurl"
+    baseurl = ["https://fake-repo.base.url"]
+    metalink = ""
+    mirrorlist = ""
+    proxy = ""
+    sslverify = True
+    gpgcheck = True
+    gpgkey = []
+
+def fakerepo_baseurl():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "name": "fake-repo-baseurl",
+        "system": False,
+        "type": "yum-baseurl",
+        "url": "https://fake-repo.base.url"
+    }
+
+class FakeSystemRepo(object):
+    id = "fake-system-repo"
+    baseurl = ["https://fake-repo.base.url"]
+    metalink = ""
+    mirrorlist = ""
+    proxy = ""
+    sslverify = True
+    gpgcheck = True
+    gpgkey = []
+
+def fakesystem_repo():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "name": "fake-system-repo",
+        "system": True,
+        "type": "yum-baseurl",
+        "url": "https://fake-repo.base.url"
+    }
+
+class FakeRepoMetalink(object):
+    id = "fake-repo-metalink"
+    baseurl = []
+    metalink = "https://fake-repo.metalink"
+    proxy = ""
+    sslverify = True
+    gpgcheck = True
+    gpgkey = []
+
+def fakerepo_metalink():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "name": "fake-repo-metalink",
+        "system": False,
+        "type": "yum-metalink",
+        "url": "https://fake-repo.metalink"
+    }
+
+class FakeRepoMirrorlist(object):
+    id = "fake-repo-mirrorlist"
+    baseurl = []
+    metalink = ""
+    mirrorlist = "https://fake-repo.mirrorlist"
+    proxy = ""
+    sslverify = True
+    gpgcheck = True
+    gpgkey = []
+
+def fakerepo_mirrorlist():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "name": "fake-repo-mirrorlist",
+        "system": False,
+        "type": "yum-mirrorlist",
+        "url": "https://fake-repo.mirrorlist"
+    }
+
+class FakeRepoProxy(object):
+    id = "fake-repo-proxy"
+    baseurl = ["https://fake-repo.base.url"]
+    metalink = ""
+    mirrorlist = ""
+    proxy = "https://fake-repo.proxy"
+    sslverify = True
+    gpgcheck = True
+    gpgkey = []
+
+def fakerepo_proxy():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "name": "fake-repo-proxy",
+        "proxy": "https://fake-repo.proxy",
+        "system": False,
+        "type": "yum-baseurl",
+        "url": "https://fake-repo.base.url"
+    }
+
+class FakeRepoGPGKey(object):
+    id = "fake-repo-gpgkey"
+    baseurl = ["https://fake-repo.base.url"]
+    metalink = ""
+    mirrorlist = ""
+    proxy = ""
+    sslverify = True
+    gpgcheck = True
+    gpgkey = ["https://fake-repo.gpgkey"]
+
+def fakerepo_gpgkey():
+    return {
+        "check_gpg": True,
+        "check_ssl": True,
+        "gpgkey_urls": [
+            "https://fake-repo.gpgkey"
+        ],
+        "name": "fake-repo-gpgkey",
+        "system": False,
+        "type": "yum-baseurl",
+        "url": "https://fake-repo.base.url"
+    }
+
+class SourceTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(self):
+        self.tmp_dir = tempfile.mkdtemp(prefix="lorax.test.repo.")
+        for f in glob("./tests/pylorax/repos/*.repo"):
+            shutil.copy2(f, self.tmp_dir)
+
+    @classmethod
+    def tearDownClass(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def test_repo_to_source_baseurl(self):
+        """Test a repo with a baseurl"""
+        self.assertEqual(repo_to_source(FakeRepoBaseUrl(), False), fakerepo_baseurl())
+
+    def test_system_repo(self):
+        """Test a system repo with a baseurl"""
+        self.assertEqual(repo_to_source(FakeSystemRepo(), True), fakesystem_repo())
+
+    def test_repo_to_source_metalink(self):
+        """Test a repo with a metalink"""
+        self.assertEqual(repo_to_source(FakeRepoMetalink, False), fakerepo_metalink())
+
+    def test_repo_to_source_mirrorlist(self):
+        """Test a repo with a mirrorlist"""
+        self.assertEqual(repo_to_source(FakeRepoMirrorlist, False), fakerepo_mirrorlist())
+
+    def test_repo_to_source_proxy(self):
+        """Test a repo with a proxy"""
+        self.assertEqual(repo_to_source(FakeRepoProxy, False), fakerepo_proxy())
+
+    def test_repo_to_source_gpgkey(self):
+        """Test a repo with a GPG key"""
+        self.assertEqual(repo_to_source(FakeRepoGPGKey, False), fakerepo_gpgkey())
+
+    def test_get_repo_sources(self):
+        """Test getting a list of sources from a repo directory"""
+        sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo"))
+        self.assertTrue("lorax-1" in sources)
+        self.assertTrue("lorax-2" in sources)
+
+    def test_delete_source_multiple(self):
+        """Test deleting a source from a repo file with multiple entries"""
+        delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "lorax-3")
+        sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo"))
+        self.assertTrue("lorax-3" not in sources)
+
+    def test_delete_source_single(self):
+        """Test deleting a source from a repo with only 1 entry"""
+        delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "single-repo")
+        sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo"))
+        self.assertTrue("single-repo" not in sources)
+        self.assertTrue(not os.path.exists(joinpaths(self.tmp_dir, "single.repo")))
+
+    def test_delete_source_other(self):
+        """Test deleting a source from a repo that doesn't match the source name"""
+        with self.assertRaises(ProjectsError):
+            delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "unknown-source")
+        sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo"))
+        self.assertTrue("lorax-1" in sources)
+        self.assertTrue("lorax-2" in sources)
+        self.assertTrue("lorax-4" in sources)
+        self.assertTrue("other-repo" in sources)
+
+    def test_source_to_repo_baseurl(self):
+        """Test creating a yum.yumRepo.YumRepository with a baseurl"""
+        repo = source_to_repo(fakerepo_baseurl())
+        self.assertEqual(repo.baseurl[0], fakerepo_baseurl()["url"])
+
+    def test_source_to_repo_metalink(self):
+        """Test creating a yum.yumRepo.YumRepository with a metalink"""
+        repo = source_to_repo(fakerepo_metalink())
+        self.assertEqual(repo.metalink, fakerepo_metalink()["url"])
+
+    def test_source_to_repo_mirrorlist(self):
+        """Test creating a yum.yumRepo.YumRepository with a mirrorlist"""
+        repo = source_to_repo(fakerepo_mirrorlist())
+        self.assertEqual(repo.mirrorlist, fakerepo_mirrorlist()["url"])
+
+    def test_source_to_repo_proxy(self):
+        """Test creating a yum.yumRepo.YumRepository with a proxy"""
+        repo = source_to_repo(fakerepo_proxy())
+        self.assertEqual(repo.proxy, fakerepo_proxy()["proxy"])
+
+    def test_source_to_repo_gpgkey(self):
+        """Test creating a yum.yumRepo.YumRepository with a proxy"""
+        repo = source_to_repo(fakerepo_gpgkey())
+        self.assertEqual(repo.gpgkey, fakerepo_gpgkey()["gpgkey_urls"])

tests/pylorax/test_server.py

@@ -47,6 +47,17 @@ class ServerTestCase(unittest.TestCase):
             raise RuntimeError("\n".join(errors))
 
         make_yum_dirs(server.config["COMPOSER_CFG"])
+
+        # copy over the test yum repositories
+        yum_repo_dir = server.config["COMPOSER_CFG"].get("composer", "repo_dir")
+        for f in glob("./tests/pylorax/repos/*.repo"):
+            shutil.copy2(f, yum_repo_dir)
+
+        # yum repo baseurl has to point to an absolute directory, so we use /tmp/lorax-empty-repo/ in the files
+        # and create an empty repository
+        os.makedirs("/tmp/lorax-empty-repo/")
+        os.system("createrepo_c /tmp/lorax-empty-repo/")
+
         yb = get_base_object(server.config["COMPOSER_CFG"])
         server.config["YUMLOCK"] = YumLock(yb=yb, lock=Lock())
 
@@ -69,6 +80,7 @@ class ServerTestCase(unittest.TestCase):
     @classmethod
     def tearDownClass(self):
         shutil.rmtree(server.config["REPO_DIR"])
+        shutil.rmtree("/tmp/lorax-empty-repo/")
 
     def test_01_status(self):
         """Test the /api/status route"""
@@ -461,6 +473,107 @@ class ServerTestCase(unittest.TestCase):
         self.assertEqual(len(deps) > 10, True)
         self.assertEqual(deps[2]["name"], "basesystem")
 
+    def test_projects_source_00_list(self):
+        """Test /api/v0/projects/source/list"""
+        resp = self.server.get("/api/v0/projects/source/list")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data["sources"], ["base", "epel", "extras", "lorax-1", "lorax-2", "lorax-3", "lorax-4", "other-repo", "single-repo", "updates"])
+
+    def test_projects_source_00_info(self):
+        """Test /api/v0/projects/source/info"""
+        resp = self.server.get("/api/v0/projects/source/info/single-repo")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        sources = data["sources"]
+        self.assertTrue("single-repo" in sources)
+
+    def test_projects_source_00_new_json(self):
+        """Test /api/v0/projects/source/new with a new json source"""
+        json_source = open("./tests/pylorax/source/test-repo.json").read()
+        self.assertTrue(len(json_source) > 0)
+        resp = self.server.post("/api/v0/projects/source/new",
+                                data=json_source,
+                                content_type="application/json")
+        data = json.loads(resp.data)
+        self.assertEqual(data, {"status":True})
+
+    def test_projects_source_00_new_toml(self):
+        """Test /api/v0/projects/source/new with a new toml source"""
+        toml_source = open("./tests/pylorax/source/test-repo.toml").read()
+        self.assertTrue(len(toml_source) > 0)
+        resp = self.server.post("/api/v0/projects/source/new",
+                                data=toml_source,
+                                content_type="text/x-toml")
+        data = json.loads(resp.data)
+        self.assertEqual(data, {"status":True})
+
+    def test_projects_source_00_replace(self):
+        """Test /api/v0/projects/source/new with a replacement source"""
+        toml_source = open("./tests/pylorax/source/replace-repo.toml").read()
+        self.assertTrue(len(toml_source) > 0)
+        resp = self.server.post("/api/v0/projects/source/new",
+                                data=toml_source,
+                                content_type="text/x-toml")
+        data = json.loads(resp.data)
+        self.assertEqual(data, {"status":True})
+
+        # Check to see if it was really changed
+        resp = self.server.get("/api/v0/projects/source/info/single-repo")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        sources = data["sources"]
+        self.assertTrue("single-repo" in sources)
+        repo = sources["single-repo"]
+        self.assertEqual(repo["check_ssl"], False)
+        self.assertTrue("gpgkey_urls" not in repo)
+
+    def test_projects_source_01_delete_system(self):
+        """Test /api/v0/projects/source/delete a system source"""
+        resp = self.server.delete("/api/v0/projects/source/delete/base")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data["status"], False)
+
+        # Make sure base is still listed
+        resp = self.server.get("/api/v0/projects/source/list")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertTrue("base" in data["sources"])
+
+    def test_projects_source_02_delete_single(self):
+        """Test /api/v0/projects/source/delete a single source"""
+        resp = self.server.delete("/api/v0/projects/source/delete/single-repo")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data, {"status":True})
+
+        # Make sure single-repo isn't listed
+        resp = self.server.get("/api/v0/projects/source/list")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertTrue("single-repo" not in data["sources"])
+
+    def test_projects_source_03_delete_unknown(self):
+        """Test /api/v0/projects/source/delete an unknown source"""
+        resp = self.server.delete("/api/v0/projects/source/delete/unknown-repo")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data["status"], False)
+
+    def test_projects_source_04_delete_multi(self):
+        """Test /api/v0/projects/source/delete a source from a file with multiple sources"""
+        resp = self.server.delete("/api/v0/projects/source/delete/lorax-3")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertEqual(data, {"status":True})
+
+        # Make sure single-repo isn't listed
+        resp = self.server.get("/api/v0/projects/source/list")
+        data = json.loads(resp.data)
+        self.assertNotEqual(data, None)
+        self.assertTrue("lorax-3" not in data["sources"])
+
     def test_modules_list(self):
         """Test /api/v0/modules/list"""
         resp = self.server.get("/api/v0/modules/list")
@@ -553,7 +666,7 @@ class ServerTestCase(unittest.TestCase):
                 return True
             if time.time() > start + 60:
                 return False
-            time.sleep(5)
+            time.sleep(1)
 
     def test_compose_01_types(self):
         """Test the /api/v0/compose/types route"""