From 6a95a314e9e423babf3bac399fbdee0e9098f401 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Tue, 6 Feb 2018 16:50:42 -0800 Subject: [PATCH] Add API routes for downloading build results Also fix a bug with the name of the queue status in the status results (it is now 'queue_status' not 'status' which is used for error responses). This adds the following routes: - /compose/metadata/ to retrieve a .tar of the build metadata - /compose/results/ to retrieve .tar of all of the build results - /compose/logs/ to retrieve a .tar of just the logs from the build - /compose/image/ to retrieve the output image from the build --- src/pylorax/api/queue.py | 65 +++++++++++++++++++++++ src/pylorax/api/v0.py | 109 +++++++++++++++++++++++++++++++++++---- 2 files changed, 164 insertions(+), 10 deletions(-) diff --git a/src/pylorax/api/queue.py b/src/pylorax/api/queue.py index 4ad4cfae..8d4113e1 100644 --- a/src/pylorax/api/queue.py +++ b/src/pylorax/api/queue.py @@ -24,6 +24,7 @@ import pytoml as toml import pwd import shutil import subprocess +from subprocess import Popen, PIPE import time from pykickstart.version import makeVersion, RHEL7 from pykickstart.parser import KickstartParser @@ -281,3 +282,67 @@ def uuid_info(cfg, uuid): "compose_type": compose_type, "queue_status": status } + +def uuid_tar(cfg, uuid, metadata=False, image=False, logs=False): + """Return a tar of the build data + + :param cfg: Configuration settings + :type cfg: ComposerConfig + :param uuid: The UUID of the build + :type uuid: str + :param metadata: Set to true to include all the metadata needed to reproduce the build + :type metadata: bool + :param image: Set to true to include the output image + :type image: bool + :param logs: Set to true to include the logs from the build + :type logs: bool + :returns: A stream of bytes from tar + :rtype: A generator + + This yields an uncompressed tar's data to the caller. It includes + the selected data to the caller by returning the Popen stdout from the tar process. + """ + uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid) + if not os.path.exists(uuid_dir): + raise RuntimeError("%s is not a valid build_id" % uuid) + + # Load the compose configuration + cfg_path = joinpaths(uuid_dir, "config.toml") + if not os.path.exists(cfg_path): + raise RuntimeError("Missing config.toml for %s" % uuid) + cfg_dict = toml.loads(open(cfg_path, "r").read()) + image_name = cfg_dict["image_name"] + + def include_file(f): + if f.endswith("/logs"): + return logs + if f.endswith(image_name): + return image + return metadata + filenames = [os.path.basename(f) for f in glob(joinpaths(uuid_dir, "*")) if include_file(f)] + + tar = Popen(["tar", "-C", uuid_dir, "-cf-"] + filenames, stdout=PIPE) + return tar.stdout + +def uuid_image(cfg, uuid): + """Return the filename and full path of the build's image file + + :param cfg: Configuration settings + :type cfg: ComposerConfig + :param uuid: The UUID of the build + :type uuid: str + :returns: The image filename and full path + :rtype: tuple of strings + """ + uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid) + if not os.path.exists(uuid_dir): + raise RuntimeError("%s is not a valid build_id" % uuid) + + # Load the compose configuration + cfg_path = joinpaths(uuid_dir, "config.toml") + if not os.path.exists(cfg_path): + raise RuntimeError("Missing config.toml for %s" % uuid) + cfg_dict = toml.loads(open(cfg_path, "r").read()) + image_name = cfg_dict["image_name"] + + return (image_name, joinpaths(uuid_dir, image_name)) diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py index 68b1604d..16f95f6b 100644 --- a/src/pylorax/api/v0.py +++ b/src/pylorax/api/v0.py @@ -625,14 +625,14 @@ POST `/api/v0/recipes/tag/` { "id": "45502a6d-06e8-48a5-a215-2b4174b3614b", "recipe": "glusterfs", - "status": "WAITING", + "queue_status": "WAITING", "timestamp": 1517362647.4570868, "version": "0.0.6" }, { "id": "6d292bd0-bec7-4825-8d7d-41ef9c3e4b73", "recipe": "kubernetes", - "status": "WAITING", + "queue_status": "WAITING", "timestamp": 1517362659.0034983, "version": "0.0.1" } @@ -641,7 +641,7 @@ POST `/api/v0/recipes/tag/` { "id": "745712b2-96db-44c0-8014-fe925c35e795", "recipe": "glusterfs", - "status": "RUNNING", + "queue_status": "RUNNING", "timestamp": 1517362633.7965999, "version": "0.0.6" } @@ -660,14 +660,14 @@ POST `/api/v0/recipes/tag/` { "id": "70b84195-9817-4b8a-af92-45e380f39894", "recipe": "glusterfs", - "status": "FINISHED", + "queue_status": "FINISHED", "timestamp": 1517351003.8210032, "version": "0.0.6" }, { "id": "e695affd-397f-4af9-9022-add2636e7459", "recipe": "glusterfs", - "status": "FINISHED", + "queue_status": "FINISHED", "timestamp": 1517362289.7193348, "version": "0.0.6" } @@ -686,7 +686,7 @@ POST `/api/v0/recipes/tag/` { "id": "8c8435ef-d6bd-4c68-9bf1-a2ef832e6b1a", "recipe": "http-server", - "status": "RUNNING", + "queue_status": "FAILED", "timestamp": 1517523249.9301329, "version": "0.0.2" } @@ -705,14 +705,14 @@ POST `/api/v0/recipes/tag/` { "id": "8c8435ef-d6bd-4c68-9bf1-a2ef832e6b1a", "recipe": "http-server", - "status": "FINISHED", + "queue_status": "FINISHED", "timestamp": 1517523644.2384307, "version": "0.0.2" }, { "id": "45502a6d-06e8-48a5-a215-2b4174b3614b", "recipe": "glusterfs", - "status": "FINISHED", + "queue_status": "FINISHED", "timestamp": 1517363442.188399, "version": "0.0.6" } @@ -782,20 +782,57 @@ DELETE `/api/v0/compose/delete/` } } +`/api/v0/compose/metadata/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Returns a .tar of the metadata used for the build. This includes all the + information needed to reproduce the build, including the final kickstart + populated with repository and package NEVRA. + + The mime type is set to 'application/x-tar' and the filename is set to + UUID-metadata.tar + + The .tar is uncompressed, but is not large. + +`/api/v0/compose/results/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Returns a .tar of the metadata, logs, and output image of the build. This + includes all the information needed to reproduce the build, including the + final kickstart populated with repository and package NEVRA. The output image + is already in compressed form so the returned tar is not compressed. + + The mime type is set to 'application/x-tar' and the filename is set to + UUID.tar + +`/api/v0/compose/logs/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Returns a .tar of the anaconda build logs. The tar is not compressed, but is + not large. + + The mime type is set to 'application/x-tar' and the filename is set to + UUID-logs.tar + +`/api/v0/compose/image/` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + Returns the output image from the build. The filename is set to the filename + from the build. eg. root.tar.xz or boot.iso. """ import logging log = logging.getLogger("lorax-composer") -from flask import jsonify, request +from flask import jsonify, request, Response, send_file from pylorax.api.compose import start_build, compose_types from pylorax.api.crossdomain import crossdomain from pylorax.api.projects import projects_list, projects_info, projects_depsolve from pylorax.api.projects import modules_list, modules_info, ProjectsError from pylorax.api.queue import queue_status, build_status, uuid_delete, uuid_status, uuid_info +from pylorax.api.queue import uuid_tar, uuid_image from pylorax.api.recipes import list_branch_files, read_recipe_commit, recipe_filename, list_commits from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe from pylorax.api.recipes import tag_recipe_commit, recipe_diff @@ -1335,7 +1372,7 @@ def v0_api(api): errors = [] for uuid in [n.strip().lower() for n in uuids.split(",")]: status = uuid_status(api.config["COMPOSER_CFG"], uuid) - if status["status"] not in ["FINISHED", "FAILED"]: + if status["queue_status"] not in ["FINISHED", "FAILED"]: errors.append({"uuid":uuid, "msg":"Build not in FINISHED or FAILED."}) else: try: @@ -1356,3 +1393,55 @@ def v0_api(api): return jsonify(status=False, msg=str(e)) return jsonify(**info) + + @api.route("/api/v0/compose/metadata/") + @crossdomain(origin="*") + def v0_compose_metadata(uuid): + """Return a tar of the metadata for the build""" + status = uuid_status(api.config["COMPOSER_CFG"], uuid) + if status["queue_status"] not in ["FINISHED", "FAILED"]: + return jsonify({"status":False, "uuid":uuid, "msg":"Build not in FINISHED or FAILED."}) + else: + return Response(uuid_tar(api.config["COMPOSER_CFG"], uuid, metadata=True, image=False, logs=False), + mimetype="application/x-tar", + headers=[("Content-Disposition", "attachment; filename=%s-metadata.tar;" % uuid)], + direct_passthrough=True) + + @api.route("/api/v0/compose/results/") + @crossdomain(origin="*") + def v0_compose_results(uuid): + """Return a tar of the metadata and the results for the build""" + status = uuid_status(api.config["COMPOSER_CFG"], uuid) + if status["queue_status"] not in ["FINISHED", "FAILED"]: + return jsonify({"status":False, "uuid":uuid, "msg":"Build not in FINISHED or FAILED."}) + else: + return Response(uuid_tar(api.config["COMPOSER_CFG"], uuid, metadata=True, image=True, logs=True), + mimetype="application/x-tar", + headers=[("Content-Disposition", "attachment; filename=%s.tar;" % uuid)], + direct_passthrough=True) + + @api.route("/api/v0/compose/logs/") + @crossdomain(origin="*") + def v0_compose_logs(uuid): + """Return a tar of the metadata for the build""" + status = uuid_status(api.config["COMPOSER_CFG"], uuid) + if status["queue_status"] not in ["FINISHED", "FAILED"]: + return jsonify({"status":False, "uuid":uuid, "msg":"Build not in FINISHED or FAILED."}) + else: + return Response(uuid_tar(api.config["COMPOSER_CFG"], uuid, metadata=False, image=False, logs=True), + mimetype="application/x-tar", + headers=[("Content-Disposition", "attachment; filename=%s-logs.tar;" % uuid)], + direct_passthrough=True) + + @api.route("/api/v0/compose/image/") + @crossdomain(origin="*") + def v0_compose_image(uuid): + """Return the output image for the build""" + status = uuid_status(api.config["COMPOSER_CFG"], uuid) + if status["queue_status"] not in ["FINISHED", "FAILED"]: + return jsonify({"status":False, "uuid":uuid, "msg":"Build not in FINISHED or FAILED."}) + else: + image_name, image_path = uuid_image(api.config["COMPOSER_CFG"], uuid) + + # XXX - Will mime type guessing work for all our output? + return send_file(image_path, as_attachment=True, attachment_filename=image_name, add_etags=False)