diff --git a/docs/lorax-composer.rst b/docs/lorax-composer.rst index 5a2d65db..737d07bb 100644 --- a/docs/lorax-composer.rst +++ b/docs/lorax-composer.rst @@ -102,3 +102,55 @@ the results directory, or it could do some post-processing on it. The end of the function should always clean up the ``./compose/`` directory, removing any unneeded extra files. This is especially true for the ``live-iso`` since it produces the contents of the iso as well as the boot.iso itself. + +Package Sources +--------------- + +By default lorax-composer uses the host's configured repositories. It copies +the ``*.repo`` files from ``/etc/yum.repos.d/`` into +``/var/lib/lorax/composer/repos.d/`` at startup, these are immutable system +repositories and cannot be deleted or changed. If you want to add additional +repos you can put them into ``/var/lib/lorax/composer/repos.d/`` or use the +``/api/v0/projects/source/*`` API routes to create them. + +The new source can be added by doing a POST to the ``/api/v0/projects/source/new`` +route using JSON (with `Content-Type` header set to `application/json`) or TOML +(with it set to `text/x-toml`). The format of the source looks like this (in +TOML):: + + name = "custom-source-1" + url = "https://url/path/to/repository/" + type = "yum-baseurl" + proxy = "https://proxy-url/" + check_ssl = true + check_gpg = true + gpgkey_urls = ["https://url/path/to/gpg-key"] + +The ``proxy`` and ``gpgkey_urls`` entries are optional. All of the others are required. The supported +types for the urls are: + +* ``yum-baseurl`` is a URL to a yum repository. +* ``yum-mirrorlist`` is a URL for a mirrorlist. +* ``yum-metalink`` is a URL for a metalink. + +If ``check_ssl`` is true the https certificates must be valid. If they are self-signed you can either set +this to false, or add your Certificate Authority to the host system. + +If ``check_gpg`` is true the GPG key must either be installed on the host system, or ``gpgkey_urls`` +should point to it. + +You can edit an existing source (other than system sources), by doing a POST to the ``new`` route +with the new version of the source. It will overwrite the previous one. + +A list of existing sources is available from ``/api/v0/projects/source/list``, and detailed info +on a source can be retrieved with the ``/api/v0/projects/source/info/`` route. By default +it returns JSON but it can also return TOML if ``?format=toml`` is added to the request. + +Non-system sources can be deleted by doing a ``DELETE`` request to the +``/api/v0/projects/source/delete/`` route. + +The documentation for the source API routes can be `found here `_ + +The configured sources are used for all blueprint depsolve operations, and for composing images. +When adding additional sources you must make sure that the packages in the source do not +conflict with any other package sources, otherwise depsolving will fail. diff --git a/src/pylorax/api/config.py b/src/pylorax/api/config.py index 2070a7d4..2258a174 100644 --- a/src/pylorax/api/config.py +++ b/src/pylorax/api/config.py @@ -44,9 +44,9 @@ def configure(conf_file="/etc/lorax/composer.conf", root_dir="/", test_config=Fa conf.add_section("composer") conf.set("composer", "share_dir", os.path.realpath(joinpaths(root_dir, "/usr/share/lorax/"))) conf.set("composer", "lib_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/composer/"))) + conf.set("composer", "repo_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/composer/repos.d/"))) conf.set("composer", "yum_conf", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/yum.conf"))) conf.set("composer", "yum_root", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/yum/root/"))) - conf.set("composer", "repo_dir", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/repos.d/"))) conf.set("composer", "cache_dir", os.path.realpath(joinpaths(root_dir, "/var/tmp/composer/cache/"))) conf.set("composer", "tmp", os.path.realpath(joinpaths(root_dir, "/var/tmp/"))) diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 5e8a5a4d..4214059d 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -17,6 +17,10 @@ import logging log = logging.getLogger("lorax-composer") +import os +from ConfigParser import ConfigParser +import yum +from glob import glob import time from yum.Errors import YumBaseError @@ -315,3 +319,175 @@ def modules_info(yb, module_names): module["dependencies"] = projects_depsolve(yb, [(module["name"], "*")]) return modules + +def repo_to_source(repo, system_source): + """Return a Weldr Source dict created from the YumRepository + + :param repo: Yum Repository + :type repo: yum.yumRepo.YumRepository + :param system_source: True if this source is an immutable system source + :type system_source: bool + :returns: A dict with Weldr Source fields filled in + :rtype: dict + + Example:: + + { + "check_gpg": true, + "check_ssl": true, + "gpgkey_url": [ + "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64" + ], + "name": "fedora", + "proxy": "http://proxy.brianlane.com:8123", + "system": true + "type": "yum-metalink", + "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64" + } + + """ + source = {"name": repo.id, "system": system_source} + if repo.baseurl: + source["url"] = repo.baseurl[0] + source["type"] = "yum-baseurl" + elif repo.metalink: + source["url"] = repo.metalink + source["type"] = "yum-metalink" + elif repo.mirrorlist: + source["url"] = repo.mirrorlist + source["type"] = "yum-mirrorlist" + else: + raise RuntimeError("Repo has no baseurl, metalink, or mirrorlist") + + # proxy is optional + if repo.proxy: + source["proxy"] = repo.proxy + + if not repo.sslverify: + source["check_ssl"] = False + else: + source["check_ssl"] = True + + if not repo.gpgcheck: + source["check_gpg"] = False + else: + source["check_gpg"] = True + + if repo.gpgkey: + source["gpgkey_urls"] = repo.gpgkey + + return source + +def source_to_repo(source): + """Return a yum YumRepository object created from a source dict + + :param source: A Weldr source dict + :type source: dict + :returns: A yum YumRepository object + :rtype: yum.yumRepo.YumRepository + + Example:: + + { + "check_gpg": True, + "check_ssl": True, + "gpgkey_urls": [ + "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64" + ], + "name": "fedora", + "proxy": "http://proxy.brianlane.com:8123", + "system": True + "type": "yum-metalink", + "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64" + } + + """ + repo = yum.yumRepo.YumRepository(source["name"]) + if source["type"] == "yum-baseurl": + repo.baseurl = [source["url"]] + elif source["type"] == "yum-metalink": + repo.metalink = source["url"] + elif source["type"] == "yum-mirrorlist": + repo.mirrorlist = source["url"] + + if "proxy" in source: + repo.proxy = source["proxy"] + + if source["check_ssl"]: + repo.sslverify = True + else: + repo.sslverify = False + + if source["check_gpg"]: + repo.gpgcheck = True + else: + repo.gpgcheck = False + + if "gpgkey_urls" in source: + repo.gpgkey = source["gpgkey_urls"] + + repo.enable() + + return repo + +def get_source_ids(source_path): + """Return a list of the source ids in a file + + :param source_path: Full path and filename of the source (yum repo) file + :type source_path: str + :returns: A list of source id strings + :rtype: list of str + """ + if not os.path.exists(source_path): + return [] + + cfg = ConfigParser() + cfg.read(source_path) + return cfg.sections() + +def get_repo_sources(source_glob): + """Return a list of sources from a directory of yum repositories + + :param source_glob: A glob to use to match the source files, including full path + :type source_glob: str + :returns: A list of the source ids in all of the matching files + :rtype: list of str + """ + sources = [] + for f in glob(source_glob): + sources.extend(get_source_ids(f)) + return sources + +def delete_repo_source(source_glob, source_name): + """Delete a source from a repo file + + :param source_glob: A glob of the repo sources to search + :type source_glob: str + :returns: None + :raises: ProjectsError if there was a problem + + A repo file may have multiple sources in it, delete only the selected source. + If it is the last one in the file, delete the file. + + WARNING: This will delete ANY source, the caller needs to ensure that a system + source_name isn't passed to it. + """ + found = False + for f in glob(source_glob): + try: + cfg = ConfigParser() + cfg.read(f) + if source_name in cfg.sections(): + found = True + cfg.remove_section(source_name) + # If there are other sections, rewrite the file without the deleted one + if len(cfg.sections()) > 0: + with open(f, "w") as cfg_file: + cfg.write(cfg_file) + else: + # No sections left, just delete the file + os.unlink(f) + except Exception as e: + raise ProjectsError("Problem deleting repo source %s: %s" % (source_name, str(e))) + if not found: + raise ProjectsError("source %s not found" % source_name) diff --git a/src/pylorax/api/queue.py b/src/pylorax/api/queue.py index e943ccd5..33fb7c4f 100644 --- a/src/pylorax/api/queue.py +++ b/src/pylorax/api/queue.py @@ -89,7 +89,7 @@ def monitor(cfg): # Pick the oldest and move it into ./run/ if not uuids: # No composes left to process, sleep for a bit - time.sleep(30) + time.sleep(5) else: src = joinpaths(cfg.composer_dir, "queue/new", uuids[0]) dst = joinpaths(cfg.composer_dir, "queue/run", uuids[0]) @@ -192,7 +192,7 @@ def make_compose(cfg, results_dir): test_path = joinpaths(results_dir, "TEST") if os.path.exists(test_path): # Pretend to run the compose - time.sleep(10) + time.sleep(5) try: test_mode = int(open(test_path, "r").read()) except Exception: diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index 0aceeec7..4047ed58 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -501,6 +501,94 @@ POST `/api/v0/blueprints/tag/` ] } +`/api/v0/projects/source/list` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Return the list of repositories used for depsolving and installing packages. + + Example:: + + { + "sources": [ + "fedora", + "fedora-cisco-openh264", + "fedora-updates-testing", + "fedora-updates" + ] + } + +`/api/v0/projects/source/info/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Return information about the comma-separated list of source names. Or all of the + sources if '*' is passed. Note that general globbing is not supported, only '*'. + + immutable system sources will have the "system" field set to true. User added sources + will have it set to false. System sources cannot be changed or deleted. + + Example:: + + { + "errors": [], + "sources": { + "fedora": { + "check_gpg": true, + "check_ssl": true, + "gpgkey_urls": [ + "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-28-x86_64" + ], + "name": "fedora", + "proxy": "http://proxy.brianlane.com:8123", + "system": true, + "type": "yum-metalink", + "url": "https://mirrors.fedoraproject.org/metalink?repo=fedora-28&arch=x86_64" + } + } + } + +POST `/api/v0/projects/source/new` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Add (or change) a source for use when depsolving blueprints and composing images. + + The ``proxy`` and ``gpgkey_urls`` entries are optional. All of the others are required. The supported + types for the urls are: + + * ``yum-baseurl`` is a URL to a yum repository. + * ``yum-mirrorlist`` is a URL for a mirrorlist. + * ``yum-metalink`` is a URL for a metalink. + + If ``check_ssl`` is true the https certificates must be valid. If they are self-signed you can either set + this to false, or add your Certificate Authority to the host system. + + If ``check_gpg`` is true the GPG key must either be installed on the host system, or ``gpgkey_urls`` + should point to it. + + You can edit an existing source (other than system sources), by doing a POST + of the new version of the source. It will overwrite the previous one. + + Example:: + + { + "name": "custom-source-1", + "url": "https://url/path/to/repository/", + "type": "yum-baseurl", + "check_ssl": true, + "check_gpg": true, + "gpgkey_urls": [ + "https://url/path/to/gpg-key" + ] + } + +DELETE `/api/v0/projects/source/delete/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Delete a user added source. This will fail if a system source is passed to + it. + + The response will be a status response with `status` set to true, or an + error response with it set to false and an error message included. + `/api/v0/modules/list[?offset=0&limit=20]` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -878,17 +966,21 @@ log = logging.getLogger("lorax-composer") import os from flask import jsonify, request, Response, send_file +import pytoml as toml +from pylorax.sysutils import joinpaths from pylorax.api.compose import start_build, compose_types from pylorax.api.crossdomain import crossdomain from pylorax.api.projects import projects_list, projects_info, projects_depsolve -from pylorax.api.projects import modules_list, modules_info, ProjectsError +from pylorax.api.projects import modules_list, modules_info, ProjectsError, repo_to_source +from pylorax.api.projects import get_repo_sources, delete_repo_source, source_to_repo from pylorax.api.queue import queue_status, build_status, uuid_delete, uuid_status, uuid_info from pylorax.api.queue import uuid_tar, uuid_image, uuid_cancel, uuid_log from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_filename, list_commits from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe from pylorax.api.recipes import tag_recipe_commit, recipe_diff from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete +from pylorax.api.yumbase import update_metadata # The API functions don't actually get called by any code here # pylint: disable=unused-variable @@ -1300,6 +1392,121 @@ def v0_api(api): return jsonify(projects=deps) + @api.route("/api/v0/projects/source/list") + @crossdomain(origin="*") + def v0_projects_source_list(): + """Return the list of source names""" + with api.config["YUMLOCK"].lock: + repos = list(api.config["YUMLOCK"].yb.repos.listEnabled()) + sources = sorted([r.id for r in repos]) + return jsonify(sources=sources) + + @api.route("/api/v0/projects/source/info/") + @crossdomain(origin="*") + def v0_projects_source_info(source_names): + """Return detailed info about the list of sources""" + out_fmt = request.args.get("format", "json") + + # Return info on all of the sources + if source_names == "*": + with api.config["YUMLOCK"].lock: + source_names = ",".join(r.id for r in api.config["YUMLOCK"].yb.repos.listEnabled()) + + sources = {} + errors = [] + system_sources = get_repo_sources("/etc/yum.repos.d/*.repo") + for source in source_names.split(","): + with api.config["YUMLOCK"].lock: + repo = api.config["YUMLOCK"].yb.repos.repos.get(source, None) + if not repo: + errors.append("%s is not a valid source" % source) + continue + sources[repo.id] = repo_to_source(repo, repo.id in system_sources) + + if out_fmt == "toml": + # With TOML output we just want to dump the raw sources, skipping the errors + return toml.dumps(sources) + else: + return jsonify(sources=sources, errors=errors) + + @api.route("/api/v0/projects/source/new", methods=["POST"]) + @crossdomain(origin="*") + def v0_projects_source_new(): + """Add a new package source. Or change an existing one""" + if request.headers['Content-Type'] == "text/x-toml": + source = toml.loads(request.data) + else: + source = request.get_json(cache=False) + + system_sources = get_repo_sources("/etc/yum.repos.d/*.repo") + if source["name"] in system_sources: + return jsonify(status=False, errors=["%s is a system source, it cannot be deleted." % source["name"]]), 400 + + try: + # Delete it from yum (if it exists) and replace it with the new one + with api.config["YUMLOCK"].lock: + yb = api.config["YUMLOCK"].yb + # If this repo already exists, delete it and replace it with the new one + repos = list(r.id for r in yb.repos.listEnabled()) + if source["name"] in repos: + yb.repos.delete(source["name"]) + + # XXX - BCL DIAGNOSTIC + repos = list(r.id for r in yb.repos.listEnabled()) + if source["name"] in repos: + return jsonify(status=False, errors=["Failed to delete Yum repo %s" % source["name"]]), 400 + + repo = source_to_repo(source) + yb.repos.add(repo) + + log.info("Updating repository metadata after adding %s", source["name"]) + update_metadata(yb) + + # Write the new repo to disk, replacing any existing ones + repo_dir = api.config["COMPOSER_CFG"].get("composer", "repo_dir") + + # Remove any previous sources with this name, ignore it if it isn't found + try: + delete_repo_source(joinpaths(repo_dir, "*.repo"), source["name"]) + except ProjectsError: + pass + + # Make sure the source name can't contain a path traversal by taking the basename + source_path = joinpaths(repo_dir, os.path.basename("%s.repo" % source["name"])) + with open(source_path, "w") as f: + f.write(str(repo)) + except Exception as e: + return jsonify(status=False, errors=[str(e)]), 400 + + return jsonify(status=True) + + @api.route("/api/v0/projects/source/delete/", methods=["DELETE"]) + @crossdomain(origin="*") + def v0_projects_source_delete(source_name): + """Delete the named source and return a status response""" + system_sources = get_repo_sources("/etc/yum.repos.d/*.repo") + if source_name in system_sources: + return jsonify(status=False, errors=["%s is a system source, it cannot be deleted." % source_name]), 400 + share_dir = api.config["COMPOSER_CFG"].get("composer", "repo_dir") + try: + # Remove the file entry for the source + delete_repo_source(joinpaths(share_dir, "*.repo"), source_name) + + # Delete the repo + with api.config["YUMLOCK"].lock: + yb = api.config["YUMLOCK"].yb + repos = list(r.id for r in yb.repos.listEnabled()) + if source_name in repos: + yb.repos.delete(source_name) + log.info("Updating repository metadata after removing %s", source_name) + update_metadata(yb) + + except ProjectsError as e: + log.error("(v0_projects_source_delete) %s", str(e)) + return jsonify(status=False, errors=[str(e)]), 400 + + return jsonify(status=True) + @api.route("/api/v0/modules/list") @api.route("/api/v0/modules/list/") @crossdomain(origin="*") diff --git a/src/pylorax/api/yumbase.py b/src/pylorax/api/yumbase.py index 4b3da3bc..317c00ac 100644 --- a/src/pylorax/api/yumbase.py +++ b/src/pylorax/api/yumbase.py @@ -103,11 +103,21 @@ def get_base_object(conf): # Update the metadata from the enabled repos to speed up later operations log.info("Updating yum repository metadata") + update_metadata(yb) + + return yb + +def update_metadata(yb): + """Update the metadata for all the enabled repos + + :param yb: The Yum base object + :type yb: yum.YumBase + :returns: None + :rtype: None + """ for r in yb.repos.sort(): r.metadata_expire = 0 r.mdpolicy = "group:all" yb.doRepoSetup() yb.repos.doSetup() yb.repos.populateSack(mdtype='all', cacheonly=1) - - return yb diff --git a/tests/pylorax/repos/multiple.repo b/tests/pylorax/repos/multiple.repo new file mode 100644 index 00000000..d68d3e2a --- /dev/null +++ b/tests/pylorax/repos/multiple.repo @@ -0,0 +1,47 @@ +[lorax-1] +name=Lorax test repo 1 +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False + +[lorax-2] +name=Lorax test repo 2 +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False + +[lorax-3] +name=Lorax test repo 3 +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False + +[lorax-4] +name=Lorax test repo 4 +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False diff --git a/tests/pylorax/repos/other.repo b/tests/pylorax/repos/other.repo new file mode 100644 index 00000000..0a3da20f --- /dev/null +++ b/tests/pylorax/repos/other.repo @@ -0,0 +1,11 @@ +[other-repo] +name=Other repo +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False diff --git a/tests/pylorax/repos/single.repo b/tests/pylorax/repos/single.repo new file mode 100644 index 00000000..c4518a01 --- /dev/null +++ b/tests/pylorax/repos/single.repo @@ -0,0 +1,11 @@ +[single-repo] +name=One repo in the file +failovermethod=priority +baseurl=file:///tmp/lorax-empty-repo/ +enabled=1 +metadata_expire=7d +repo_gpgcheck=0 +type=rpm +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch +skip_if_unavailable=False diff --git a/tests/pylorax/source/replace-repo.toml b/tests/pylorax/source/replace-repo.toml new file mode 100644 index 00000000..d0b1ef8d --- /dev/null +++ b/tests/pylorax/source/replace-repo.toml @@ -0,0 +1,6 @@ +name = "single-repo" +url = "file:///tmp/lorax-empty-repo/" +type = "yum-baseurl" +check_ssl = false +check_gpg = true +gpgkey_urls = [] diff --git a/tests/pylorax/source/test-repo.json b/tests/pylorax/source/test-repo.json new file mode 100644 index 00000000..579c87a3 --- /dev/null +++ b/tests/pylorax/source/test-repo.json @@ -0,0 +1 @@ +{"name": "new-repo-1", "url": "file:///tmp/lorax-empty-repo/", "type": "yum-baseurl", "check_ssl": true, "check_gpg": true, "gpgkey_urls": ["file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch"]} diff --git a/tests/pylorax/source/test-repo.toml b/tests/pylorax/source/test-repo.toml new file mode 100644 index 00000000..2cc5e262 --- /dev/null +++ b/tests/pylorax/source/test-repo.toml @@ -0,0 +1,6 @@ +name = "new-repo-2" +url = "file:///tmp/lorax-empty-repo/" +type = "yum-baseurl" +check_ssl = true +check_gpg = true +gpgkey_urls = ["file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch"] diff --git a/tests/pylorax/test_projects.py b/tests/pylorax/test_projects.py index 037469ea..e9a61b0a 100644 --- a/tests/pylorax/test_projects.py +++ b/tests/pylorax/test_projects.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from glob import glob import os import mock import time @@ -23,11 +24,13 @@ import unittest from yum.Errors import YumBaseError +from pylorax.sysutils import joinpaths from pylorax.api.config import configure, make_yum_dirs from pylorax.api.projects import api_time, api_changelog, yaps_to_project, yaps_to_project_info from pylorax.api.projects import tm_to_dep, yaps_to_module, projects_list, projects_info, projects_depsolve from pylorax.api.projects import modules_list, modules_info, ProjectsError, dep_evra, dep_nevra from pylorax.api.yumbase import get_base_object +from pylorax.api.projects import repo_to_source, get_repo_sources, delete_repo_source, source_to_repo class Yaps(object): @@ -247,3 +250,215 @@ class ConfigureTest(unittest.TestCase): def test_configure_reads_non_existing_file(self): config = configure(conf_file=self.conf_file + '.non-existing') self.assertEqual(config.get('composer', 'cache_dir'), '/var/tmp/composer/cache') + +class FakeRepoBaseUrl(object): + id = "fake-repo-baseurl" + baseurl = ["https://fake-repo.base.url"] + metalink = "" + mirrorlist = "" + proxy = "" + sslverify = True + gpgcheck = True + gpgkey = [] + +def fakerepo_baseurl(): + return { + "check_gpg": True, + "check_ssl": True, + "name": "fake-repo-baseurl", + "system": False, + "type": "yum-baseurl", + "url": "https://fake-repo.base.url" + } + +class FakeSystemRepo(object): + id = "fake-system-repo" + baseurl = ["https://fake-repo.base.url"] + metalink = "" + mirrorlist = "" + proxy = "" + sslverify = True + gpgcheck = True + gpgkey = [] + +def fakesystem_repo(): + return { + "check_gpg": True, + "check_ssl": True, + "name": "fake-system-repo", + "system": True, + "type": "yum-baseurl", + "url": "https://fake-repo.base.url" + } + +class FakeRepoMetalink(object): + id = "fake-repo-metalink" + baseurl = [] + metalink = "https://fake-repo.metalink" + proxy = "" + sslverify = True + gpgcheck = True + gpgkey = [] + +def fakerepo_metalink(): + return { + "check_gpg": True, + "check_ssl": True, + "name": "fake-repo-metalink", + "system": False, + "type": "yum-metalink", + "url": "https://fake-repo.metalink" + } + +class FakeRepoMirrorlist(object): + id = "fake-repo-mirrorlist" + baseurl = [] + metalink = "" + mirrorlist = "https://fake-repo.mirrorlist" + proxy = "" + sslverify = True + gpgcheck = True + gpgkey = [] + +def fakerepo_mirrorlist(): + return { + "check_gpg": True, + "check_ssl": True, + "name": "fake-repo-mirrorlist", + "system": False, + "type": "yum-mirrorlist", + "url": "https://fake-repo.mirrorlist" + } + +class FakeRepoProxy(object): + id = "fake-repo-proxy" + baseurl = ["https://fake-repo.base.url"] + metalink = "" + mirrorlist = "" + proxy = "https://fake-repo.proxy" + sslverify = True + gpgcheck = True + gpgkey = [] + +def fakerepo_proxy(): + return { + "check_gpg": True, + "check_ssl": True, + "name": "fake-repo-proxy", + "proxy": "https://fake-repo.proxy", + "system": False, + "type": "yum-baseurl", + "url": "https://fake-repo.base.url" + } + +class FakeRepoGPGKey(object): + id = "fake-repo-gpgkey" + baseurl = ["https://fake-repo.base.url"] + metalink = "" + mirrorlist = "" + proxy = "" + sslverify = True + gpgcheck = True + gpgkey = ["https://fake-repo.gpgkey"] + +def fakerepo_gpgkey(): + return { + "check_gpg": True, + "check_ssl": True, + "gpgkey_urls": [ + "https://fake-repo.gpgkey" + ], + "name": "fake-repo-gpgkey", + "system": False, + "type": "yum-baseurl", + "url": "https://fake-repo.base.url" + } + +class SourceTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.tmp_dir = tempfile.mkdtemp(prefix="lorax.test.repo.") + for f in glob("./tests/pylorax/repos/*.repo"): + shutil.copy2(f, self.tmp_dir) + + @classmethod + def tearDownClass(self): + shutil.rmtree(self.tmp_dir) + + def test_repo_to_source_baseurl(self): + """Test a repo with a baseurl""" + self.assertEqual(repo_to_source(FakeRepoBaseUrl(), False), fakerepo_baseurl()) + + def test_system_repo(self): + """Test a system repo with a baseurl""" + self.assertEqual(repo_to_source(FakeSystemRepo(), True), fakesystem_repo()) + + def test_repo_to_source_metalink(self): + """Test a repo with a metalink""" + self.assertEqual(repo_to_source(FakeRepoMetalink, False), fakerepo_metalink()) + + def test_repo_to_source_mirrorlist(self): + """Test a repo with a mirrorlist""" + self.assertEqual(repo_to_source(FakeRepoMirrorlist, False), fakerepo_mirrorlist()) + + def test_repo_to_source_proxy(self): + """Test a repo with a proxy""" + self.assertEqual(repo_to_source(FakeRepoProxy, False), fakerepo_proxy()) + + def test_repo_to_source_gpgkey(self): + """Test a repo with a GPG key""" + self.assertEqual(repo_to_source(FakeRepoGPGKey, False), fakerepo_gpgkey()) + + def test_get_repo_sources(self): + """Test getting a list of sources from a repo directory""" + sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo")) + self.assertTrue("lorax-1" in sources) + self.assertTrue("lorax-2" in sources) + + def test_delete_source_multiple(self): + """Test deleting a source from a repo file with multiple entries""" + delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "lorax-3") + sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo")) + self.assertTrue("lorax-3" not in sources) + + def test_delete_source_single(self): + """Test deleting a source from a repo with only 1 entry""" + delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "single-repo") + sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo")) + self.assertTrue("single-repo" not in sources) + self.assertTrue(not os.path.exists(joinpaths(self.tmp_dir, "single.repo"))) + + def test_delete_source_other(self): + """Test deleting a source from a repo that doesn't match the source name""" + with self.assertRaises(ProjectsError): + delete_repo_source(joinpaths(self.tmp_dir, "*.repo"), "unknown-source") + sources = get_repo_sources(joinpaths(self.tmp_dir, "*.repo")) + self.assertTrue("lorax-1" in sources) + self.assertTrue("lorax-2" in sources) + self.assertTrue("lorax-4" in sources) + self.assertTrue("other-repo" in sources) + + def test_source_to_repo_baseurl(self): + """Test creating a yum.yumRepo.YumRepository with a baseurl""" + repo = source_to_repo(fakerepo_baseurl()) + self.assertEqual(repo.baseurl[0], fakerepo_baseurl()["url"]) + + def test_source_to_repo_metalink(self): + """Test creating a yum.yumRepo.YumRepository with a metalink""" + repo = source_to_repo(fakerepo_metalink()) + self.assertEqual(repo.metalink, fakerepo_metalink()["url"]) + + def test_source_to_repo_mirrorlist(self): + """Test creating a yum.yumRepo.YumRepository with a mirrorlist""" + repo = source_to_repo(fakerepo_mirrorlist()) + self.assertEqual(repo.mirrorlist, fakerepo_mirrorlist()["url"]) + + def test_source_to_repo_proxy(self): + """Test creating a yum.yumRepo.YumRepository with a proxy""" + repo = source_to_repo(fakerepo_proxy()) + self.assertEqual(repo.proxy, fakerepo_proxy()["proxy"]) + + def test_source_to_repo_gpgkey(self): + """Test creating a yum.yumRepo.YumRepository with a proxy""" + repo = source_to_repo(fakerepo_gpgkey()) + self.assertEqual(repo.gpgkey, fakerepo_gpgkey()["gpgkey_urls"]) diff --git a/tests/pylorax/test_server.py b/tests/pylorax/test_server.py index 06b480b7..87947873 100644 --- a/tests/pylorax/test_server.py +++ b/tests/pylorax/test_server.py @@ -47,6 +47,17 @@ class ServerTestCase(unittest.TestCase): raise RuntimeError("\n".join(errors)) make_yum_dirs(server.config["COMPOSER_CFG"]) + + # copy over the test yum repositories + yum_repo_dir = server.config["COMPOSER_CFG"].get("composer", "repo_dir") + for f in glob("./tests/pylorax/repos/*.repo"): + shutil.copy2(f, yum_repo_dir) + + # yum repo baseurl has to point to an absolute directory, so we use /tmp/lorax-empty-repo/ in the files + # and create an empty repository + os.makedirs("/tmp/lorax-empty-repo/") + os.system("createrepo_c /tmp/lorax-empty-repo/") + yb = get_base_object(server.config["COMPOSER_CFG"]) server.config["YUMLOCK"] = YumLock(yb=yb, lock=Lock()) @@ -69,6 +80,7 @@ class ServerTestCase(unittest.TestCase): @classmethod def tearDownClass(self): shutil.rmtree(server.config["REPO_DIR"]) + shutil.rmtree("/tmp/lorax-empty-repo/") def test_01_status(self): """Test the /api/status route""" @@ -461,6 +473,107 @@ class ServerTestCase(unittest.TestCase): self.assertEqual(len(deps) > 10, True) self.assertEqual(deps[2]["name"], "basesystem") + def test_projects_source_00_list(self): + """Test /api/v0/projects/source/list""" + resp = self.server.get("/api/v0/projects/source/list") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data["sources"], ["base", "epel", "extras", "lorax-1", "lorax-2", "lorax-3", "lorax-4", "other-repo", "single-repo", "updates"]) + + def test_projects_source_00_info(self): + """Test /api/v0/projects/source/info""" + resp = self.server.get("/api/v0/projects/source/info/single-repo") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + sources = data["sources"] + self.assertTrue("single-repo" in sources) + + def test_projects_source_00_new_json(self): + """Test /api/v0/projects/source/new with a new json source""" + json_source = open("./tests/pylorax/source/test-repo.json").read() + self.assertTrue(len(json_source) > 0) + resp = self.server.post("/api/v0/projects/source/new", + data=json_source, + content_type="application/json") + data = json.loads(resp.data) + self.assertEqual(data, {"status":True}) + + def test_projects_source_00_new_toml(self): + """Test /api/v0/projects/source/new with a new toml source""" + toml_source = open("./tests/pylorax/source/test-repo.toml").read() + self.assertTrue(len(toml_source) > 0) + resp = self.server.post("/api/v0/projects/source/new", + data=toml_source, + content_type="text/x-toml") + data = json.loads(resp.data) + self.assertEqual(data, {"status":True}) + + def test_projects_source_00_replace(self): + """Test /api/v0/projects/source/new with a replacement source""" + toml_source = open("./tests/pylorax/source/replace-repo.toml").read() + self.assertTrue(len(toml_source) > 0) + resp = self.server.post("/api/v0/projects/source/new", + data=toml_source, + content_type="text/x-toml") + data = json.loads(resp.data) + self.assertEqual(data, {"status":True}) + + # Check to see if it was really changed + resp = self.server.get("/api/v0/projects/source/info/single-repo") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + sources = data["sources"] + self.assertTrue("single-repo" in sources) + repo = sources["single-repo"] + self.assertEqual(repo["check_ssl"], False) + self.assertTrue("gpgkey_urls" not in repo) + + def test_projects_source_01_delete_system(self): + """Test /api/v0/projects/source/delete a system source""" + resp = self.server.delete("/api/v0/projects/source/delete/base") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data["status"], False) + + # Make sure base is still listed + resp = self.server.get("/api/v0/projects/source/list") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertTrue("base" in data["sources"]) + + def test_projects_source_02_delete_single(self): + """Test /api/v0/projects/source/delete a single source""" + resp = self.server.delete("/api/v0/projects/source/delete/single-repo") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data, {"status":True}) + + # Make sure single-repo isn't listed + resp = self.server.get("/api/v0/projects/source/list") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertTrue("single-repo" not in data["sources"]) + + def test_projects_source_03_delete_unknown(self): + """Test /api/v0/projects/source/delete an unknown source""" + resp = self.server.delete("/api/v0/projects/source/delete/unknown-repo") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data["status"], False) + + def test_projects_source_04_delete_multi(self): + """Test /api/v0/projects/source/delete a source from a file with multiple sources""" + resp = self.server.delete("/api/v0/projects/source/delete/lorax-3") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertEqual(data, {"status":True}) + + # Make sure single-repo isn't listed + resp = self.server.get("/api/v0/projects/source/list") + data = json.loads(resp.data) + self.assertNotEqual(data, None) + self.assertTrue("lorax-3" not in data["sources"]) + def test_modules_list(self): """Test /api/v0/modules/list""" resp = self.server.get("/api/v0/modules/list") @@ -553,7 +666,7 @@ class ServerTestCase(unittest.TestCase): return True if time.time() > start + 60: return False - time.sleep(5) + time.sleep(1) def test_compose_01_types(self): """Test the /api/v0/compose/types route"""