almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

Add support for composer-cli compose commands.

Browse Source 
This adds all the commands needed to start, monitor, delete, and
download a compose.
This commit is contained in:
Brian C. Lane 2018-03-13 11:24:07 -07:00
parent 9ba24f305d
commit 56766e794f
7 changed files with 503 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/bin/composer-cli
View File

@ -31,7 +31,17 @@ VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
# Documentation for the commands
epilog = """
compose tar <recipe> Depsolve Recipe and compose a tar file using export from bdcs
compose start <recipe> <type> Start a compose using the selected recipe and output type.
types List the supported output types.
status List the status of all running and finished composes.
log <uuid> [<size>kB] Show the last 1kB of the compose log.
cancel <uuid> Cancel a running compose and delete any intermediate results.
delete <uuid,...> Delete the listed compose results.
info <uuid> Show detailed information on the compose.
metadata <uuid> Download the metadata use to create the compose to <uuid>-metadata.tar
logs <uuid> Download the compose logs to <uuid>-logs.tar
results <uuid> Download all of the compose results; metadata, logs, and image to <uuid>.tar
image <uuid> Download the output image from the compose. Filename depends on the type.
recipes list List the names of the available recipes.
show <recipe,...> Display the recipe in TOML format.
changes <recipe,...> Display the changes for each recipe.

15
src/composer/cli/__init__.py
View File

@ -39,11 +39,14 @@ def main(opts):
:param opts: Cmdline arguments
:type opts: argparse.Namespace
"""
if len(opts.args) > 0 and opts.args[0] in command_map:
return command_map[opts.args[0]](opts)
elif len(opts.args) == 0:
log.error("Unknown command: %s", opts.args)
if len(opts.args) == 0:
log.error("Missing command")
return 1
elif opts.args[0] not in command_map:
log.error("Unknown command %s", opts.args[0])
return 1
if len(opts.args) == 1:
log.error("Missing %s sub-command", opts.args[0])
return 1
else:
log.error("Unknown command: %s", opts.args)
return 1
return command_map[opts.args[0]](opts)

382
src/composer/cli/compose.py
View File

@ -14,6 +14,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
log = logging.getLogger("composer-cli")
import sys
import json
from composer import http_client as client
from composer.cli.utilities import argify, handle_api_result, packageNEVRA
def compose_cmd(opts):
"""Process compose commands
@ -22,5 +30,377 @@ def compose_cmd(opts):
:type opts: argparse.Namespace
:returns: Value to return from sys.exit()
:rtype: int
This dispatches the compose commands to a function
"""
return 1
cmd_map = {
"status": compose_status,
"types": compose_types,
"start": compose_start,
"log": compose_log,
"cancel": compose_cancel,
"delete": compose_delete,
"details": compose_details,
"metadata": compose_metadata,
"results": compose_results,
"logs": compose_logs,
"image": compose_image,
}
if opts.args[1] not in cmd_map:
log.error("Unknown compose command: %s", opts.args[1])
return 1
return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
def compose_status(socket_path, api_version, args, show_json=False):
"""Return the status of all known composes
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
This doesn't map directly to an API command, it combines the results from queue, finished,
and failed so raw JSON output is not available.
"""
def get_status(compose):
return {"id": compose["id"],
"recipe": compose["recipe"],
"version": compose["version"],
"status": compose["queue_status"]}
# Sort the status in a specific order
def sort_status(a):
order = ["RUNNING", "WAITING", "FINISHED", "FAILED"]
return (order.index(a["status"]), a["recipe"], a["version"])
status = []
# Get the composes currently in the queue
api_route = client.api_url(api_version, "/compose/queue")
result = client.get_url_json(socket_path, api_route)
status.extend(map(get_status, result["run"] + result["new"]))
# Get the list of finished composes
api_route = client.api_url(api_version, "/compose/finished")
result = client.get_url_json(socket_path, api_route)
status.extend(map(get_status, result["finished"]))
# Get the list of failed composes
api_route = client.api_url(api_version, "/compose/failed")
result = client.get_url_json(socket_path, api_route)
status.extend(map(get_status, result["failed"]))
# Sort them by status (running, waiting, finished, failed) and then by name and version.
status.sort(key=sort_status)
if show_json:
print(json.dumps(status, indent=4))
return 0
# Print them as UUID RECIPE STATUS
for c in status:
print("%s %-8s %-15s %s" % (c["id"], c["status"], c["recipe"], c["version"]))
def compose_types(socket_path, api_version, args, show_json=False):
"""Return information about the supported compose types
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
Add additional details to types that are known to composer-cli. Raw JSON output does not
include this extra information.
"""
api_route = client.api_url(api_version, "/compose/types")
result = client.get_url_json(socket_path, api_route)
if show_json:
print(json.dumps(result, indent=4))
return 0
print("Compose Types: " + ", ".join([t["name"] for t in result["types"]]))
def compose_start(socket_path, api_version, args, show_json=False):
"""Start a new compose using the selected recipe and type
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose start <recipe-name> <compose-type>
"""
if len(args) == 0:
log.error("start is missing the recipe name and output type")
return 1
if len(args) == 1:
log.error("start is missing the output type")
return 1
config = {
"recipe_name": args[0],
"compose_type": args[1],
"branch": "master"
}
api_route = client.api_url(api_version, "/compose")
result = client.post_url_json(socket_path, api_route, json.dumps(config))
if show_json:
print(json.dumps(result, indent=4))
return 0
if result.get("error", False):
log.error(result["error"]["msg"])
return 1
if result["status"] == False:
return 1
print("Compose %s added to the queue" % result["build_id"])
return 0
def compose_log(socket_path, api_version, args, show_json=False):
"""Show the last part of the compose log
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose log <uuid> [<size>kB]
This will display the last 1kB of the compose's log file. Can be used to follow progress
during the build.
"""
if len(args) == 0:
log.error("log is missing the compose build id")
return 1
if len(args) == 2:
try:
log_size = int(args[1])
except ValueError:
log.error("Log size must be an integer.")
return 1
else:
log_size = 1024
api_route = client.api_url(api_version, "/compose/log/%s?size=%d" % (args[0], log_size))
result = client.get_url_raw(socket_path, api_route)
print(result)
def compose_cancel(socket_path, api_version, args, show_json=False):
"""Cancel a running compose
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose cancel <uuid>
This will cancel a running compose. It does nothing if the compose has finished.
"""
if len(args) == 0:
log.error("cancel is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/cancel/%s" % args[0])
result = client.delete_url_json(socket_path, api_route)
return handle_api_result(result, show_json)
def compose_delete(socket_path, api_version, args, show_json=False):
"""Delete a finished compose's results
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose delete <uuid,...>
Delete the listed compose results. It will only delete results for composes that have finished
or failed, not a running compose.
"""
if len(args) == 0:
log.error("delete is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/delete/%s" % (",".join(argify(args))))
result = client.delete_url_json(socket_path, api_route)
if show_json:
print(json.dumps(result, indent=4))
return 0
# Print any errors
for err in result.get("errors", []):
log.error("%s: %s", err["uuid"], err["msg"])
if result.get("errors", []):
return 1
else:
return 0
def compose_details(socket_path, api_version, args, show_json=False):
"""Return detailed information about the compose
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose details <uuid>
This returns information about the compose, including the recipe and the dependencies.
"""
if len(args) == 0:
log.error("details is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/info/%s" % args[0])
result = client.get_url_json(socket_path, api_route)
if show_json:
print(json.dumps(result, indent=4))
return 0
print("%s %-8s %-15s %s %s" % (result["id"],
result["queue_status"],
result["recipe"]["name"],
result["recipe"]["version"],
result["compose_type"]))
print("Recipe Packages:")
for p in result["recipe"]["packages"]:
print(" %s-%s" % (p["name"], p["version"]))
print("Recipe Modules:")
for m in result["recipe"]["modules"]:
print(" %s-%s" % (m["name"], m["version"]))
print("Dependencies:")
for d in result["deps"]["packages"]:
print(" " + packageNEVRA(d))
def compose_metadata(socket_path, api_version, args, show_json=False):
"""Download a tar file of the compose's metadata
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose metadata <uuid>
Saves the metadata as uuid-metadata.tar
"""
if len(args) == 0:
log.error("metadata is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/metadata/%s" % args[0])
return client.download_file(socket_path, api_route)
def compose_results(socket_path, api_version, args, show_json=False):
"""Download a tar file of the compose's results
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose results <uuid>
The results includes the metadata, output image, and logs.
It is saved as uuid.tar
"""
if len(args) == 0:
log.error("results is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/results/%s" % args[0])
return client.download_file(socket_path, api_route, sys.stdout.isatty())
def compose_logs(socket_path, api_version, args, show_json=False):
"""Download a tar of the compose's logs
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose logs <uuid>
Saves the logs as uuid-logs.tar
"""
if len(args) == 0:
log.error("logs is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/logs/%s" % args[0])
return client.download_file(socket_path, api_route, sys.stdout.isatty())
def compose_image(socket_path, api_version, args, show_json=False):
"""Download the compose's output image
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param api_version: Version of the API to talk to. eg. "0"
:type api_version: str
:param args: List of remaining arguments from the cmdline
:type args: list of str
:param show_json: Set to True to show the JSON output instead of the human readable output
:type show_json: bool
compose image <uuid>
This downloads only the result image, saving it as the image name, which depends on the type
of compose that was selected.
"""
if len(args) == 0:
log.error("logs is missing the compose build id")
return 1
api_route = client.api_url(api_version, "/compose/image/%s" % args[0])
return client.download_file(socket_path, api_route, sys.stdout.isatty())

4
src/composer/cli/projects.py
View File

@ -34,6 +34,10 @@ def projects_cmd(opts):
"list": projects_list,
"info": projects_info,
}
if opts.args[1] not in cmd_map:
log.error("Unknown projects command: %s", opts.args[1])
return 1
return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
def projects_list(socket_path, api_version, args, show_json=False):

18
src/composer/cli/recipes.py
View File

@ -22,6 +22,7 @@ import json
from composer import http_client as client
from composer.cli.utilities import argify, frozen_toml_filename, toml_filename, handle_api_result
from composer.cli.utilities import packageNEVRA
def recipes_cmd(opts):
"""Process recipes commands
@ -47,6 +48,10 @@ def recipes_cmd(opts):
"undo": recipes_undo,
"workspace": recipes_workspace
}
if opts.args[1] not in cmd_map:
log.error("Unknown recipes command: %s", opts.args[1])
return 1
return cmd_map[opts.args[1]](opts.socket, opts.api_version, opts.args[2:], opts.json)
def recipes_list(socket_path, api_version, args, show_json=False):
@ -295,19 +300,6 @@ def recipes_depsolve(socket_path, api_version, args, show_json=False):
return 0
def packageNEVRA(pkg):
"""Return the package info as a NEVRA
:param pkg: The package details
:type pkg: dict
:returns: name-[epoch:]version-release-arch
:rtype: str
"""
if pkg["epoch"]:
return "%s-%s:%s-%s.%s" % (pkg["name"], pkg["epoch"], pkg["version"], pkg["release"], pkg["arch"])
else:
return "%s-%s-%s.%s" % (pkg["name"], pkg["version"], pkg["release"], pkg["arch"])
def recipes_push(socket_path, api_version, args, show_json=False):
"""Push a recipe TOML file to the server, updating the recipe

13
src/composer/cli/utilities.py
View File

@ -68,3 +68,16 @@ def handle_api_result(result, show_json=False):
return 0
else:
return 1
def packageNEVRA(pkg):
"""Return the package info as a NEVRA
:param pkg: The package details
:type pkg: dict
:returns: name-[epoch:]version-release-arch
:rtype: str
"""
if pkg["epoch"]:
return "%s-%s:%s-%s.%s" % (pkg["name"], pkg["epoch"], pkg["version"], pkg["release"], pkg["arch"])
else:
return "%s-%s-%s.%s" % (pkg["name"], pkg["version"], pkg["release"], pkg["arch"])

80
src/composer/http_client.py
View File

@ -18,6 +18,7 @@ import logging
log = logging.getLogger("composer-cli")
import os
import sys
import json
from composer.unix_socket import UnixHTTPConnectionPool
@ -104,3 +105,82 @@ def post_url_toml(socket_path, url, body):
body=body.encode("utf-8"),
headers={"Content-Type": "text/x-toml"})
return json.loads(r.data.decode("utf-8"))
def post_url_json(socket_path, url, body):
"""POST some JSON data to the URL
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param url: URL to send POST to
:type url: str
:param body: The data for the body of the POST
:type body: str
:returns: The json response from the server
:rtype: dict
"""
http = UnixHTTPConnectionPool(socket_path)
r = http.request("POST", url,
body=body.encode("utf-8"),
headers={"Content-Type": "application/json"})
return json.loads(r.data.decode("utf-8"))
def get_filename(response):
"""Get the filename from the response header
:param response: The urllib3 response object
:type response: Response
:raises: RuntimeError if it cannot find a filename in the header
:returns: Filename from content-disposition header
:rtype: str
"""
log.debug("Headers = %s", response.headers)
if "content-disposition" not in response.headers:
raise RuntimeError("No Content-Disposition header; cannot get filename")
try:
k, _, v = response.headers["content-disposition"].split(";")[1].strip().partition("=")
if k != "filename":
raise RuntimeError("No filename= found in content-disposition header")
except RuntimeError:
raise
except Exception as e:
raise RuntimeError("Error parsing filename from content-disposition header: %s" % str(e))
return os.path.basename(v)
def download_file(socket_path, url, progress=True):
"""Download a file, saving it to the CWD with the included filename
:param socket_path: Path to the Unix socket to use for API communication
:type socket_path: str
:param url: URL to send POST to
:type url: str
"""
http = UnixHTTPConnectionPool(socket_path)
r = http.request("GET", url, preload_content=False)
filename = get_filename(r)
if os.path.exists(filename):
msg = "%s exists, skipping download" % filename
log.error(msg)
raise RuntimeError(msg)
with open(filename, "wb") as f:
while True:
data = r.read(10 * 1024**2)
if not data:
break
f.write(data)
if progress:
data_written = f.tell()
if data_written > 5 * 1024**2:
sys.stdout.write("%s: %0.2f MB \r" % (filename, data_written / 1024**2))
else:
sys.stdout.write("%s: %0.2f kB\r" % (filename, data_written / 1024))
sys.stdout.flush()
print("")
r.release_conn()
return 0