Add support for composer-cli compose commands.

This adds all the commands needed to start, monitor, delete, and
download a compose.

src/bin/composer-cli

@@ -31,7 +31,17 @@ VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
 
 # Documentation for the commands
 epilog = """
-compose tar <recipe>             Depsolve Recipe and compose a tar file using export from bdcs
+compose start <recipe> <type>    Start a compose using the selected recipe and output type.
+        types                    List the supported output types.
+        status                   List the status of all running and finished composes.
+        log <uuid> [<size>kB]    Show the last 1kB of the compose log.
+        cancel <uuid>            Cancel a running compose and delete any intermediate results.
+        delete <uuid,...>        Delete the listed compose results.
+        info <uuid>              Show detailed information on the compose.
+        metadata <uuid>          Download the metadata use to create the compose to <uuid>-metadata.tar
+        logs <uuid>              Download the compose logs to <uuid>-logs.tar
+        results <uuid>           Download all of the compose results; metadata, logs, and image to <uuid>.tar
+        image <uuid>             Download the output image from the compose. Filename depends on the type.
 recipes list                     List the names of the available recipes.
         show <recipe,...>        Display the recipe in TOML format.
         changes <recipe,...>     Display the changes for each recipe.

src/composer/cli/__init__.py

@@ -39,11 +39,14 @@ def main(opts):
     :param opts: Cmdline arguments
     :type opts: argparse.Namespace
     """
-    if len(opts.args) > 0 and opts.args[0] in command_map:
+    if len(opts.args) == 0:
-        return command_map[opts.args[0]](opts)
+        log.error("Missing command")
-    elif len(opts.args) == 0:
+        return 1
-        log.error("Unknown command: %s", opts.args)
+    elif opts.args[0] not in command_map:
+        log.error("Unknown command %s", opts.args[0])
+        return 1
+    if len(opts.args) == 1:
+        log.error("Missing %s sub-command", opts.args[0])
         return 1
     else:
-        log.error("Unknown command: %s", opts.args)
+        return command_map[opts.args[0]](opts)
-        return 1

src/composer/cli/compose.py

@@ -14,6 +14,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
+import logging
+log = logging.getLogger("composer-cli")
+
+import sys
+import json
+
+from composer import http_client as client
+from composer.cli.utilities import argify, handle_api_result, packageNEVRA
 
 def compose_cmd(opts):
     """Process compose commands
@@ -22,5 +30,377 @@ def compose_cmd(opts):
     :type opts: argparse.Namespace
     :returns: Value to return from sys.exit()
     :rtype: int
+
+    This dispatches the compose commands to a function
     """
-    return 1
+    cmd_map = {
+        "status":   compose_status,
+        "types":    compose_types,
+        "start":    compose_start,
+        "log":      compose_log,
+        "cancel":   compose_cancel,
+        "delete":   compose_delete,
+        "details":  compose_details,
+        "metadata": compose_metadata,
+        "results":  compose_results,
+        "logs":     compose_logs,
+        "image":    compose_image,
+        }
+    if opts.args[1] not in cmd_map:
+        log.error("Unknown compose command: %s", opts.args[1])
+        return 1
+
+    return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
+
+def compose_status(socket_path, api_version, args, show_json=False):
+    """Return the status of all known composes
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    This doesn't map directly to an API command, it combines the results from queue, finished,
+    and failed so raw JSON output is not available.
+    """
+    def get_status(compose):
+        return {"id": compose["id"],
+                "recipe": compose["recipe"],
+                "version": compose["version"],
+                "status": compose["queue_status"]}
+
+    # Sort the status in a specific order
+    def sort_status(a):
+        order = ["RUNNING", "WAITING", "FINISHED", "FAILED"]
+        return (order.index(a["status"]), a["recipe"], a["version"])
+
+    status = []
+
+    # Get the composes currently in the queue
+    api_route = client.api_url(api_version, "/compose/queue")
+    result = client.get_url_json(socket_path, api_route)
+    status.extend(map(get_status, result["run"] + result["new"]))
+
+    # Get the list of finished composes
+    api_route = client.api_url(api_version, "/compose/finished")
+    result = client.get_url_json(socket_path, api_route)
+    status.extend(map(get_status, result["finished"]))
+
+    # Get the list of failed composes
+    api_route = client.api_url(api_version, "/compose/failed")
+    result = client.get_url_json(socket_path, api_route)
+    status.extend(map(get_status, result["failed"]))
+
+    # Sort them by status (running, waiting, finished, failed) and then by name and version.
+    status.sort(key=sort_status)
+
+    if show_json:
+        print(json.dumps(status, indent=4))
+        return 0
+
+    # Print them as UUID RECIPE STATUS
+    for c in status:
+        print("%s %-8s %-15s %s" % (c["id"], c["status"], c["recipe"], c["version"]))
+
+
+def compose_types(socket_path, api_version, args, show_json=False):
+    """Return information about the supported compose types
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    Add additional details to types that are known to composer-cli. Raw JSON output does not
+    include this extra information.
+    """
+    api_route = client.api_url(api_version, "/compose/types")
+    result = client.get_url_json(socket_path, api_route)
+    if show_json:
+        print(json.dumps(result, indent=4))
+        return 0
+
+    print("Compose Types: " + ", ".join([t["name"] for t in result["types"]]))
+
+def compose_start(socket_path, api_version, args, show_json=False):
+    """Start a new compose using the selected recipe and type
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose start <recipe-name> <compose-type>
+    """
+    if len(args) == 0:
+        log.error("start is missing the recipe name and output type")
+        return 1
+    if len(args) == 1:
+        log.error("start is missing the output type")
+        return 1
+
+    config = {
+        "recipe_name": args[0],
+        "compose_type": args[1],
+        "branch": "master"
+        }
+    api_route = client.api_url(api_version, "/compose")
+    result = client.post_url_json(socket_path, api_route, json.dumps(config))
+
+    if show_json:
+        print(json.dumps(result, indent=4))
+        return 0
+
+    if result.get("error", False):
+        log.error(result["error"]["msg"])
+        return 1
+
+    if result["status"] == False:
+        return 1
+
+    print("Compose %s added to the queue" % result["build_id"])
+    return 0
+
+def compose_log(socket_path, api_version, args, show_json=False):
+    """Show the last part of the compose log
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose log <uuid> [<size>kB]
+
+    This will display the last 1kB of the compose's log file. Can be used to follow progress
+    during the build.
+    """
+    if len(args) == 0:
+        log.error("log is missing the compose build id")
+        return 1
+    if len(args) == 2:
+        try:
+            log_size = int(args[1])
+        except ValueError:
+            log.error("Log size must be an integer.")
+            return 1
+    else:
+        log_size = 1024
+
+    api_route = client.api_url(api_version, "/compose/log/%s?size=%d" % (args[0], log_size))
+    result = client.get_url_raw(socket_path, api_route)
+
+    print(result)
+
+def compose_cancel(socket_path, api_version, args, show_json=False):
+    """Cancel a running compose
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose cancel <uuid>
+
+    This will cancel a running compose. It does nothing if the compose has finished.
+    """
+    if len(args) == 0:
+        log.error("cancel is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/cancel/%s" % args[0])
+    result = client.delete_url_json(socket_path, api_route)
+    return handle_api_result(result, show_json)
+
+def compose_delete(socket_path, api_version, args, show_json=False):
+    """Delete a finished compose's results
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose delete <uuid,...>
+
+    Delete the listed compose results. It will only delete results for composes that have finished
+    or failed, not a running compose.
+    """
+    if len(args) == 0:
+        log.error("delete is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/delete/%s" % (",".join(argify(args))))
+    result = client.delete_url_json(socket_path, api_route)
+
+    if show_json:
+        print(json.dumps(result, indent=4))
+        return 0
+
+    # Print any errors
+    for err in result.get("errors", []):
+        log.error("%s: %s", err["uuid"], err["msg"])
+
+    if result.get("errors", []):
+        return 1
+    else:
+        return 0
+
+def compose_details(socket_path, api_version, args, show_json=False):
+    """Return detailed information about the compose
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose details <uuid>
+
+    This returns information about the compose, including the recipe and the dependencies.
+    """
+    if len(args) == 0:
+        log.error("details is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/info/%s" % args[0])
+    result = client.get_url_json(socket_path, api_route)
+    if show_json:
+        print(json.dumps(result, indent=4))
+        return 0
+
+    print("%s %-8s %-15s %s %s" % (result["id"], 
+                                   result["queue_status"],
+                                   result["recipe"]["name"],
+                                   result["recipe"]["version"],
+                                   result["compose_type"]))
+    print("Recipe Packages:")
+    for p in result["recipe"]["packages"]:
+        print("    %s-%s" % (p["name"], p["version"]))
+
+    print("Recipe Modules:")
+    for m in result["recipe"]["modules"]:
+        print("    %s-%s" % (m["name"], m["version"]))
+
+    print("Dependencies:")
+    for d in result["deps"]["packages"]:
+        print("    " + packageNEVRA(d))
+
+def compose_metadata(socket_path, api_version, args, show_json=False):
+    """Download a tar file of the compose's metadata
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose metadata <uuid>
+
+    Saves the metadata as uuid-metadata.tar
+    """
+    if len(args) == 0:
+        log.error("metadata is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/metadata/%s" % args[0])
+    return client.download_file(socket_path, api_route)
+
+def compose_results(socket_path, api_version, args, show_json=False):
+    """Download a tar file of the compose's results
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose results <uuid>
+
+    The results includes the metadata, output image, and logs.
+    It is saved as uuid.tar
+    """
+    if len(args) == 0:
+        log.error("results is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/results/%s" % args[0])
+    return client.download_file(socket_path, api_route, sys.stdout.isatty())
+
+def compose_logs(socket_path, api_version, args, show_json=False):
+    """Download a tar of the compose's logs
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose logs <uuid>
+
+    Saves the logs as uuid-logs.tar
+    """
+    if len(args) == 0:
+        log.error("logs is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/logs/%s" % args[0])
+    return client.download_file(socket_path, api_route, sys.stdout.isatty())
+
+def compose_image(socket_path, api_version, args, show_json=False):
+    """Download the compose's output image
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param api_version: Version of the API to talk to. eg. "0"
+    :type api_version: str
+    :param args: List of remaining arguments from the cmdline
+    :type args: list of str
+    :param show_json: Set to True to show the JSON output instead of the human readable output
+    :type show_json: bool
+
+    compose image <uuid>
+
+    This downloads only the result image, saving it as the image name, which depends on the type
+    of compose that was selected.
+    """
+    if len(args) == 0:
+        log.error("logs is missing the compose build id")
+        return 1
+
+    api_route = client.api_url(api_version, "/compose/image/%s" % args[0])
+    return client.download_file(socket_path, api_route, sys.stdout.isatty())

src/composer/cli/projects.py

@@ -34,6 +34,10 @@ def projects_cmd(opts):
         "list":      projects_list,
         "info":      projects_info,
         }
+    if opts.args[1] not in cmd_map:
+        log.error("Unknown projects command: %s", opts.args[1])
+        return 1
+
     return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
 
 def projects_list(socket_path, api_version, args, show_json=False):

src/composer/cli/recipes.py

@@ -22,6 +22,7 @@ import json
 
 from composer import http_client as client
 from composer.cli.utilities import argify, frozen_toml_filename, toml_filename, handle_api_result
+from composer.cli.utilities import packageNEVRA
 
 def recipes_cmd(opts):
     """Process recipes commands
@@ -47,6 +48,10 @@ def recipes_cmd(opts):
         "undo":      recipes_undo,
         "workspace": recipes_workspace
         }
+    if opts.args[1] not in cmd_map:
+        log.error("Unknown recipes command: %s", opts.args[1])
+        return 1
+
     return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
 
 def recipes_list(socket_path, api_version, args, show_json=False):
@@ -295,19 +300,6 @@ def recipes_depsolve(socket_path, api_version, args, show_json=False):
 
     return 0
 
-def packageNEVRA(pkg):
-    """Return the package info as a NEVRA
-
-    :param pkg: The package details
-    :type pkg: dict
-    :returns: name-[epoch:]version-release-arch
-    :rtype: str
-    """
-    if pkg["epoch"]:
-        return "%s-%s:%s-%s.%s" % (pkg["name"], pkg["epoch"], pkg["version"], pkg["release"], pkg["arch"])
-    else:
-        return "%s-%s-%s.%s" % (pkg["name"], pkg["version"], pkg["release"], pkg["arch"])
-
 def recipes_push(socket_path, api_version, args, show_json=False):
     """Push a recipe TOML file to the server, updating the recipe
 

src/composer/cli/utilities.py

@@ -68,3 +68,16 @@ def handle_api_result(result, show_json=False):
         return 0
     else:
         return 1
+
+def packageNEVRA(pkg):
+    """Return the package info as a NEVRA
+
+    :param pkg: The package details
+    :type pkg: dict
+    :returns: name-[epoch:]version-release-arch
+    :rtype: str
+    """
+    if pkg["epoch"]:
+        return "%s-%s:%s-%s.%s" % (pkg["name"], pkg["epoch"], pkg["version"], pkg["release"], pkg["arch"])
+    else:
+        return "%s-%s-%s.%s" % (pkg["name"], pkg["version"], pkg["release"], pkg["arch"])

src/composer/http_client.py

@@ -18,6 +18,7 @@ import logging
 log = logging.getLogger("composer-cli")
 
 import os
+import sys
 import json
 
 from composer.unix_socket import UnixHTTPConnectionPool
@@ -104,3 +105,82 @@ def post_url_toml(socket_path, url, body):
                      body=body.encode("utf-8"),
                      headers={"Content-Type": "text/x-toml"})
     return json.loads(r.data.decode("utf-8"))
+
+def post_url_json(socket_path, url, body):
+    """POST some JSON data to the URL
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param url: URL to send POST to
+    :type url: str
+    :param body: The data for the body of the POST
+    :type body: str
+    :returns: The json response from the server
+    :rtype: dict
+    """
+    http = UnixHTTPConnectionPool(socket_path)
+    r = http.request("POST", url,
+                     body=body.encode("utf-8"),
+                     headers={"Content-Type": "application/json"})
+    return json.loads(r.data.decode("utf-8"))
+
+def get_filename(response):
+    """Get the filename from the response header
+
+    :param response: The urllib3 response object
+    :type response: Response
+    :raises: RuntimeError if it cannot find a filename in the header
+    :returns: Filename from content-disposition header
+    :rtype: str
+    """
+    log.debug("Headers = %s", response.headers)
+    if "content-disposition" not in response.headers:
+        raise RuntimeError("No Content-Disposition header; cannot get filename")
+
+    try:
+        k, _, v = response.headers["content-disposition"].split(";")[1].strip().partition("=")
+        if k != "filename":
+            raise RuntimeError("No filename= found in content-disposition header")
+    except RuntimeError:
+        raise
+    except Exception as e:
+        raise RuntimeError("Error parsing filename from content-disposition header: %s" % str(e))
+
+    return os.path.basename(v)
+
+def download_file(socket_path, url, progress=True):
+    """Download a file, saving it to the CWD with the included filename
+
+    :param socket_path: Path to the Unix socket to use for API communication
+    :type socket_path: str
+    :param url: URL to send POST to
+    :type url: str
+    """
+    http = UnixHTTPConnectionPool(socket_path)
+    r = http.request("GET", url, preload_content=False)
+
+    filename = get_filename(r)
+    if os.path.exists(filename):
+        msg = "%s exists, skipping download" % filename
+        log.error(msg)
+        raise RuntimeError(msg)
+
+    with open(filename, "wb") as f:
+        while True:
+            data = r.read(10 * 1024**2)
+            if not data:
+                break
+            f.write(data)
+
+            if progress:
+                data_written = f.tell()
+                if data_written > 5 * 1024**2:
+                    sys.stdout.write("%s: %0.2f MB    \r" % (filename, data_written / 1024**2))
+                else:
+                    sys.stdout.write("%s: %0.2f kB\r" % (filename, data_written / 1024))
+                sys.stdout.flush()
+
+    print("")
+    r.release_conn()
+
+    return 0