diff --git a/setup.py b/setup.py
index f2e4a033..dc33ac14 100644
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@ setup(name="lorax",
url="http://www.github.com/weldr/lorax/",
download_url="http://www.github.com/weldr/lorax/releases/",
license="GPLv2+",
- packages=["pylorax", "pylorax.api", "composer", "composer.cli"],
+ packages=["pylorax", "pylorax.api", "composer", "composer.cli", "lifted"],
package_dir={"" : "src"},
data_files=data_files
)
diff --git a/share/lifted/providers/azure/playbook.yaml b/share/lifted/providers/azure/playbook.yaml
new file mode 100644
index 00000000..6dc8ccb0
--- /dev/null
+++ b/share/lifted/providers/azure/playbook.yaml
@@ -0,0 +1,47 @@
+- hosts: localhost
+ connection: local
+ tasks:
+ - name: Make sure provided credentials work and the storage account exists
+ azure_rm_storageaccount_facts:
+ subscription_id: "{{ subscription_id }}"
+ client_id: "{{ client_id }}"
+ secret: "{{ secret }}"
+ tenant: "{{ tenant }}"
+ resource_group: "{{ resource_group }}"
+ name: "{{ storage_account_name }}"
+ register: storageaccount_facts
+ - name: Fail if we couldn't log in or the storage account was not found
+ fail:
+ msg: "Invalid credentials or storage account not found!"
+ when: storageaccount_facts.ansible_facts.azure_storageaccounts | length < 1
+ - stat:
+ path: "{{ image_path }}"
+ register: image_stat
+ - set_fact:
+ image_id: "{{ image_name }}-{{ image_stat['stat']['checksum'] }}.vhd"
+ - name: Upload image to Azure
+ azure_rm_storageblob:
+ subscription_id: "{{ subscription_id }}"
+ client_id: "{{ client_id }}"
+ secret: "{{ secret }}"
+ tenant: "{{ tenant }}"
+ resource_group: "{{ resource_group }}"
+ storage_account_name: "{{ storage_account_name }}"
+ container: "{{ storage_container }}"
+ src: "{{ image_path }}"
+ blob: "{{ image_id }}"
+ blob_type: page
+ force: no
+ - set_fact:
+ host: "{{ storage_account_name }}.blob.core.windows.net"
+ - name: Import image
+ azure_rm_image:
+ subscription_id: "{{ subscription_id }}"
+ client_id: "{{ client_id }}"
+ secret: "{{ secret }}"
+ tenant: "{{ tenant }}"
+ resource_group: "{{ resource_group }}"
+ name: "{{ image_name }}"
+ os_type: Linux
+ location: "{{ location }}"
+ source: "https://{{ host }}/{{ storage_container }}/{{ image_id }}"
diff --git a/share/lifted/providers/azure/provider.toml b/share/lifted/providers/azure/provider.toml
new file mode 100644
index 00000000..27909834
--- /dev/null
+++ b/share/lifted/providers/azure/provider.toml
@@ -0,0 +1,53 @@
+display = "Azure"
+
+supported_types = [
+ "vhd",
+]
+
+[settings-info.resource_group]
+display = "Resource group"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.storage_account_name]
+display = "Storage account name"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.storage_container]
+display = "Storage container"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.subscription_id]
+display = "Subscription ID"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.client_id]
+display = "Client ID"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.secret]
+display = "Secret"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.tenant]
+display = "Tenant"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.location]
+display = "Location"
+type = "string"
+placeholder = ""
+regex = ''
diff --git a/share/lifted/providers/dummy/playbook.yaml b/share/lifted/providers/dummy/playbook.yaml
new file mode 100644
index 00000000..f4dbce43
--- /dev/null
+++ b/share/lifted/providers/dummy/playbook.yaml
@@ -0,0 +1,4 @@
+- hosts: localhost
+ connection: local
+ tasks:
+ - pause: seconds=30
diff --git a/share/lifted/providers/dummy/provider.toml b/share/lifted/providers/dummy/provider.toml
new file mode 100644
index 00000000..f5b7880a
--- /dev/null
+++ b/share/lifted/providers/dummy/provider.toml
@@ -0,0 +1,4 @@
+display = "Dummy"
+
+[settings-info]
+# This provider has no settings.
diff --git a/share/lifted/providers/openstack/playbook.yaml b/share/lifted/providers/openstack/playbook.yaml
new file mode 100644
index 00000000..bac3fec7
--- /dev/null
+++ b/share/lifted/providers/openstack/playbook.yaml
@@ -0,0 +1,20 @@
+- hosts: localhost
+ connection: local
+ tasks:
+ - stat:
+ path: "{{ image_path }}"
+ register: image_stat
+ - set_fact:
+ image_id: "{{ image_name }}-{{ image_stat['stat']['checksum'] }}.qcow2"
+ - name: Upload image to OpenStack
+ os_image:
+ auth:
+ auth_url: "{{ auth_url }}"
+ username: "{{ username }}"
+ password: "{{ password }}"
+ project_name: "{{ project_name }}"
+ os_user_domain_name: "{{ user_domain_name }}"
+ os_project_domain_name: "{{ project_domain_name }}"
+ name: "{{ image_id }}"
+ filename: "{{ image_path }}"
+ is_public: "{{ is_public }}"
diff --git a/share/lifted/providers/openstack/provider.toml b/share/lifted/providers/openstack/provider.toml
new file mode 100644
index 00000000..2b51f7bf
--- /dev/null
+++ b/share/lifted/providers/openstack/provider.toml
@@ -0,0 +1,45 @@
+display = "OpenStack"
+
+supported_types = [
+ "qcow2",
+]
+
+[settings-info.auth_url]
+display = "Authentication URL"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.username]
+display = "Username"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.password]
+display = "Password"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.project_name]
+display = "Project name"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.user_domain_name]
+display = "User domain name"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.project_domain_name]
+display = "Project domain name"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.is_public]
+display = "Allow public access"
+type = "boolean"
diff --git a/share/lifted/providers/vsphere/playbook.yaml b/share/lifted/providers/vsphere/playbook.yaml
new file mode 100644
index 00000000..fbc92f4b
--- /dev/null
+++ b/share/lifted/providers/vsphere/playbook.yaml
@@ -0,0 +1,17 @@
+- hosts: localhost
+ connection: local
+ tasks:
+ - stat:
+ path: "{{ image_path }}"
+ register: image_stat
+ - set_fact:
+ image_id: "{{ image_name }}-{{ image_stat['stat']['checksum'] }}.vmdk"
+ - name: Upload image to vSphere
+ vsphere_copy:
+ login: "{{ username }}"
+ password: "{{ password }}"
+ host: "{{ host }}"
+ datacenter: "{{ datacenter }}"
+ datastore: "{{ datastore }}"
+ src: "{{ image_path }}"
+ path: "{{ folder }}/{{ image_id }}"
diff --git a/share/lifted/providers/vsphere/provider.toml b/share/lifted/providers/vsphere/provider.toml
new file mode 100644
index 00000000..85ac61ea
--- /dev/null
+++ b/share/lifted/providers/vsphere/provider.toml
@@ -0,0 +1,42 @@
+display = "vSphere"
+
+supported_types = [
+ "vmdk",
+]
+
+[settings-info.datacenter]
+display = "Datacenter"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.datastore]
+display = "Datastore"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.host]
+display = "Host"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.folder]
+display = "Folder"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.username]
+display = "Username"
+type = "string"
+placeholder = ""
+regex = ''
+
+[settings-info.password]
+display = "Password"
+type = "string"
+placeholder = ""
+regex = ''
+
diff --git a/src/lifted/__init__.py b/src/lifted/__init__.py
new file mode 100644
index 00000000..171c52fd
--- /dev/null
+++ b/src/lifted/__init__.py
@@ -0,0 +1,16 @@
+#
+# Copyright (C) 2019 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
diff --git a/src/lifted/providers.py b/src/lifted/providers.py
new file mode 100644
index 00000000..19fadf83
--- /dev/null
+++ b/src/lifted/providers.py
@@ -0,0 +1,172 @@
+#
+# Copyright (C) 2019 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+from glob import glob
+import os
+import re
+import stat
+
+import pylorax.api.toml as toml
+
+
+def resolve_provider(ucfg, provider_name):
+ """Get information about the specified provider as defined in that
+ provider's `provider.toml`, including the provider's display name and expected
+ settings.
+
+ At a minimum, each setting has a display name (that likely differs from its
+ snake_case name) and a type. Currently, there are two types of settings:
+ string and boolean. String settings can optionally have a "placeholder"
+ value for use on the front end and a "regex" for making sure that a value
+ follows an expected pattern.
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: the name of the provider to look for
+ :type provider_name: str
+ :raises: RuntimeError when the provider couldn't be found
+ :returns: the provider
+ :rtype: dict
+ """
+ path = os.path.join(ucfg["providers_dir"], provider_name, "provider.toml")
+ try:
+ with open(path) as provider_file:
+ provider = toml.load(provider_file)
+ except OSError as error:
+ raise RuntimeError(f'Couldn\'t find provider "{provider_name}"!') from error
+
+ return provider
+
+
+def load_profiles(ucfg, provider_name):
+ """Return all settings profiles associated with a provider
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: name a provider to find profiles for
+ :type provider_name: str
+ :returns: a dict of settings dicts, keyed by profile name
+ :rtype: dict
+ """
+
+ def load_path(path):
+ with open(path) as file:
+ return toml.load(file)
+
+ def get_name(path):
+ return os.path.splitext(os.path.basename(path))[0]
+
+ paths = glob(os.path.join(ucfg["settings_dir"], provider_name, "*"))
+ return {get_name(path): load_path(path) for path in paths}
+
+
+def resolve_playbook_path(ucfg, provider_name):
+ """Given a provider's name, return the path to its playbook
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: the name of the provider to find the playbook for
+ :type provider_name: str
+ :raises: RuntimeError when the provider couldn't be found
+ :returns: the path to the playbook
+ :rtype: str
+ """
+ path = os.path.join(ucfg["providers_dir"], provider_name, "playbook.yaml")
+ if not os.path.isfile(path):
+ raise RuntimeError(f'Couldn\'t find playbook for "{provider_name}"!')
+ return path
+
+
+def list_providers(ucfg):
+ """List the names of the available upload providers
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :returns: a list of all available provider_names
+ :rtype: list of str
+ """
+ paths = glob(os.path.join(ucfg["providers_dir"], "*"))
+ return [os.path.basename(path) for path in paths]
+
+
+def validate_settings(ucfg, provider_name, settings, image_name=None):
+ """Raise a ValueError if any settings are invalid
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: the name of the provider to validate the settings
+ against
+ :type provider_name: str
+ :param settings: the settings to validate
+ :type settings: dict
+ :param image_name: optionally check whether an image_name is valid
+ :type image_name: str
+ :raises: ValueError when the passed settings are invalid
+ :raises: RuntimeError when provider_name can't be found
+ """
+ if image_name == "":
+ raise ValueError("Image name cannot be empty!")
+ type_map = {"string": str, "boolean": bool}
+ settings_info = resolve_provider(ucfg, provider_name)["settings-info"]
+ for key, value in settings.items():
+ if key not in settings_info:
+ raise ValueError(f'Received unexpected setting: "{key}"!')
+ setting_type = settings_info[key]["type"]
+ correct_type = type_map[setting_type]
+ if not isinstance(value, correct_type):
+ raise ValueError(
+ f'Expected a {correct_type} for "{key}", received a {type(value)}!'
+ )
+ if setting_type == "string" and "regex" in settings_info[key]:
+ if not re.match(settings_info[key]["regex"], value):
+ raise ValueError(f'Value "{value}" is invalid for setting "{key}"!')
+
+
+def save_settings(ucfg, provider_name, profile, settings):
+ """Save (and overwrite) settings for a given provider
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: the name of the cloud provider, e.g. "azure"
+ :type provider_name: str
+ :param profile: the name of the profile to save
+ :type profile: str != ""
+ :param settings: settings to save for that provider
+ :type settings: dict
+ :raises: ValueError when passed invalid settings or an invalid profile name
+ """
+ if not profile:
+ raise ValueError("Profile name cannot be empty!")
+ validate_settings(ucfg, provider_name, settings, image_name=None)
+
+ directory = os.path.join(ucfg["settings_dir"], provider_name)
+
+ # create the settings directory if it doesn't exist
+ os.makedirs(directory, exist_ok=True)
+
+ path = os.path.join(directory, f"{profile}.toml")
+ # touch the TOML file if it doesn't exist
+ if not os.path.isfile(path):
+ open(path, "a").close()
+
+ # make sure settings files aren't readable by others, as they will contain
+ # sensitive credentials
+ current = stat.S_IMODE(os.lstat(path).st_mode)
+ os.chmod(path, current & ~stat.S_IROTH)
+
+ with open(path, "w") as settings_file:
+ toml.dump(settings, settings_file)
diff --git a/src/lifted/queue.py b/src/lifted/queue.py
new file mode 100644
index 00000000..a025ef97
--- /dev/null
+++ b/src/lifted/queue.py
@@ -0,0 +1,270 @@
+#
+# Copyright (C) 2019 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+from functools import partial
+from glob import glob
+import logging
+import multiprocessing
+
+# We use a multiprocessing Pool for uploads so that we can cancel them with a
+# simple SIGINT, which should bubble down to subprocesses.
+from multiprocessing import Pool
+
+# multiprocessing.dummy is to threads as multiprocessing is to processes.
+# Since daemonic processes can't have children, we use a thread to monitor the
+# upload pool.
+from multiprocessing.dummy import Process
+
+from operator import attrgetter
+import os
+import stat
+import time
+
+import pylorax.api.toml as toml
+
+from lifted.upload import Upload
+from lifted.providers import resolve_playbook_path, validate_settings
+
+# the maximum number of simultaneous uploads
+SIMULTANEOUS_UPLOADS = 1
+
+log = logging.getLogger("lifted")
+multiprocessing.log_to_stderr().setLevel(logging.INFO)
+
+
+def _get_queue_path(ucfg):
+ path = ucfg["queue_dir"]
+
+ # create the upload_queue directory if it doesn't exist
+ os.makedirs(path, exist_ok=True)
+
+ return path
+
+
+def _get_upload_path(ucfg, uuid, write=False):
+ path = os.path.join(_get_queue_path(ucfg), f"{uuid}.toml")
+ if write and not os.path.exists(path):
+ open(path, "a").close()
+ if os.path.exists(path):
+ # make sure uploads aren't readable by others, as they will contain
+ # sensitive credentials
+ current = stat.S_IMODE(os.lstat(path).st_mode)
+ os.chmod(path, current & ~stat.S_IROTH)
+ return path
+
+
+def _list_upload_uuids(ucfg):
+ paths = glob(os.path.join(_get_queue_path(ucfg), "*"))
+ return [os.path.splitext(os.path.basename(path))[0] for path in paths]
+
+
+def _write_upload(ucfg, upload):
+ with open(_get_upload_path(ucfg, upload.uuid, write=True), "w") as upload_file:
+ toml.dump(upload.serializable(), upload_file)
+
+
+def _write_callback(ucfg):
+ return partial(_write_upload, ucfg)
+
+
+def get_upload(ucfg, uuid, ignore_missing=False, ignore_corrupt=False):
+ """Get an Upload object by UUID
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param uuid: UUID of the upload to get
+ :type uuid: str
+ :param ignore_missing: if True, don't raise a RuntimeError when the
+ specified upload is missing, instead just return None
+ :type ignore_missing: bool
+ :param ignore_corrupt: if True, don't raise a RuntimeError when the
+ specified upload could not be deserialized, instead just return None
+ :type ignore_corrupt: bool
+ :returns: the upload object or None
+ :rtype: Upload or None
+ :raises: RuntimeError
+ """
+ try:
+ with open(_get_upload_path(ucfg, uuid), "r") as upload_file:
+ return Upload(**toml.load(upload_file))
+ except FileNotFoundError as error:
+ if not ignore_missing:
+ raise RuntimeError(f"Could not find upload {uuid}!") from error
+ except toml.TomlError as error:
+ if not ignore_corrupt:
+ raise RuntimeError(f"Could not parse upload {uuid}!") from error
+
+
+def get_uploads(ucfg, uuids):
+ """Gets a list of Upload objects from a list of upload UUIDs, ignoring
+ missing or corrupt uploads
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param uuids: list of upload UUIDs to get
+ :type uuids: list of str
+ :returns: a list of the uploads that were successfully deserialized
+ :rtype: list of Upload
+ """
+ uploads = (
+ get_upload(ucfg, uuid, ignore_missing=True, ignore_corrupt=True)
+ for uuid in uuids
+ )
+ return list(filter(None, uploads))
+
+
+def get_all_uploads(ucfg):
+ """Get a list of all stored Upload objects
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :returns: a list of all stored upload objects
+ :rtype: list of Upload
+ """
+ return get_uploads(ucfg, _list_upload_uuids(ucfg))
+
+
+def create_upload(ucfg, provider_name, image_name, settings):
+ """Creates a new upload
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param provider_name: the name of the cloud provider to upload to, e.g.
+ "azure"
+ :type provider_name: str
+ :param image_name: what to name the image in the cloud
+ :type image_name: str
+ :param settings: settings to pass to the upload, specific to the cloud
+ provider
+ :type settings: dict
+ :returns: the created upload object
+ :rtype: Upload
+ """
+ validate_settings(ucfg, provider_name, settings, image_name)
+ return Upload(
+ provider_name=provider_name,
+ playbook_path=resolve_playbook_path(ucfg, provider_name),
+ image_name=image_name,
+ settings=settings,
+ status_callback=_write_callback(ucfg),
+ )
+
+
+def ready_upload(ucfg, uuid, image_path):
+ """Pass an image_path to an upload and mark it ready to execute
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param uuid: the UUID of the upload to mark ready
+ :type uuid: str
+ :param image_path: the path of the image to pass to the upload
+ :type image_path: str
+ """
+ get_upload(ucfg, uuid).ready(image_path, _write_callback(ucfg))
+
+
+def reset_upload(ucfg, uuid, new_image_name=None, new_settings=None):
+ """Reset an upload so it can be attempted again
+
+ :param ucfg: upload config
+ :type ucfg: object
+ :param uuid: the UUID of the upload to reset
+ :type uuid: str
+ :param new_image_name: optionally update the upload's image_name
+ :type new_image_name: str
+ :param new_settings: optionally update the upload's settings
+ :type new_settings: dict
+ """
+ upload = get_upload(ucfg, uuid)
+ validate_settings(
+ ucfg,
+ upload.provider_name,
+ new_settings or upload.settings,
+ new_image_name or upload.image_name,
+ )
+ if new_image_name:
+ upload.image_name = new_image_name
+ if new_settings:
+ upload.settings = new_settings
+ upload.reset(_write_callback(ucfg))
+
+
+def cancel_upload(ucfg, uuid):
+ """Cancel an upload
+
+ :param ucfg: the compose config
+ :type ucfg: ComposerConfig
+ :param uuid: the UUID of the upload to cancel
+ :type uuid: str
+ """
+ get_upload(ucfg, uuid).cancel(_write_callback(ucfg))
+
+
+def delete_upload(ucfg, uuid):
+ """Delete an upload
+
+ :param ucfg: the compose config
+ :type ucfg: ComposerConfig
+ :param uuid: the UUID of the upload to delete
+ :type uuid: str
+ """
+ upload = get_upload(ucfg, uuid)
+ if upload and upload.is_cancellable():
+ upload.cancel()
+ os.remove(_get_upload_path(ucfg, uuid))
+
+
+def start_upload_monitor(ucfg):
+ """Start a thread that manages the upload queue
+
+ :param ucfg: the compose config
+ :type ucfg: ComposerConfig
+ """
+ process = Process(target=_monitor, args=(ucfg,))
+ process.daemon = True
+ process.start()
+
+
+def _monitor(ucfg):
+ log.info("Started upload monitor.")
+ for upload in get_all_uploads(ucfg):
+ # Set abandoned uploads to FAILED
+ if upload.status == "RUNNING":
+ upload.set_status("FAILED", _write_callback(ucfg))
+ pool = Pool(processes=SIMULTANEOUS_UPLOADS)
+ pool_uuids = set()
+
+ def remover(uuid):
+ return lambda _: pool_uuids.remove(uuid)
+
+ while True:
+ # Every second, scoop up READY uploads from the filesystem and throw
+ # them in the pool
+ all_uploads = get_all_uploads(ucfg)
+ for upload in sorted(all_uploads, key=attrgetter("creation_time")):
+ ready = upload.status == "READY"
+ if ready and upload.uuid not in pool_uuids:
+ log.info("Starting upload %s...", upload.uuid)
+ pool_uuids.add(upload.uuid)
+ callback = remover(upload.uuid)
+ pool.apply_async(
+ upload.execute,
+ (_write_callback(ucfg),),
+ callback=callback,
+ error_callback=callback,
+ )
+ time.sleep(1)
diff --git a/src/lifted/upload.py b/src/lifted/upload.py
new file mode 100644
index 00000000..ddd90af6
--- /dev/null
+++ b/src/lifted/upload.py
@@ -0,0 +1,188 @@
+#
+# Copyright (C) 2018-2019 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+
+from datetime import datetime
+from enum import Enum
+from functools import partial
+import logging
+from multiprocessing import current_process
+import os
+import signal
+from uuid import uuid4
+
+from ansible_runner.interface import run as ansible_run
+
+log = logging.getLogger("lifted")
+
+
+class Upload:
+ """Represents an upload of an image to a cloud provider. Instances of this
+ class are serialized as TOML and stored in the upload queue directory,
+ which is /var/lib/lorax/upload/queue/ by default"""
+
+ def __init__(
+ self,
+ uuid=None,
+ provider_name=None,
+ playbook_path=None,
+ image_name=None,
+ settings=None,
+ creation_time=None,
+ upload_log=None,
+ upload_pid=None,
+ image_path=None,
+ status_callback=None,
+ status=None,
+ ):
+ self.uuid = uuid or str(uuid4())
+ self.provider_name = provider_name
+ self.playbook_path = playbook_path
+ self.image_name = image_name
+ self.settings = settings
+ self.creation_time = creation_time or datetime.now().timestamp()
+ self.upload_log = upload_log or ""
+ self.upload_pid = upload_pid
+ self.image_path = image_path
+ if status:
+ self.status = status
+ else:
+ self.set_status("WAITING", status_callback)
+
+ def _log(self, message, callback=None):
+ """Logs something to the upload log with an optional callback
+
+ :param message: the object to log
+ :type message: object
+ :param callback: a function of the form callback(self)
+ :type callback: function
+ """
+ log.info(str(message))
+ self.upload_log += f"{message}\n"
+ if callback:
+ callback(self)
+
+ def serializable(self):
+ """Returns a representation of the object as a dict for serialization
+
+ :returns: the object's __dict__
+ :rtype: dict
+ """
+ return self.__dict__
+
+ def summary(self):
+ """Return a dict with useful information about the upload
+
+ :returns: upload information
+ :rtype: dict
+ """
+
+ return {
+ "uuid": self.uuid,
+ "status": self.status,
+ "provider_name": self.provider_name,
+ "image_name": self.image_name,
+ "image_path": self.image_path,
+ "creation_time": self.creation_time,
+ "settings": self.settings,
+ }
+
+ def set_status(self, status, status_callback=None):
+ """Sets the status of the upload with an optional callback
+
+ :param status: the new status
+ :type status: str
+ :param status_callback: a function of the form callback(self)
+ :type status_callback: function
+ """
+ self.status = status
+ if status_callback:
+ status_callback(self)
+
+ def ready(self, image_path, status_callback):
+ """Provide an image_path and mark the upload as ready to execute
+
+ :param image_path: path of the image to upload
+ :type image_path: str
+ :param status_callback: a function of the form callback(self)
+ :type status_callback: function
+ """
+ self.image_path = image_path
+ if self.status == "WAITING":
+ self.set_status("READY", status_callback)
+
+ def reset(self, status_callback):
+ """Reset the upload so it can be attempted again
+
+ :param status_callback: a function of the form callback(self)
+ :type status_callback: function
+ """
+ if self.is_cancellable():
+ raise RuntimeError(f"Can't reset, status is {self.status}!")
+ if not self.image_path:
+ raise RuntimeError(f"Can't reset, no image supplied yet!")
+ # self.error = None
+ self._log("Resetting...")
+ self.set_status("READY", status_callback)
+
+ def is_cancellable(self):
+ """Is the upload in a cancellable state?
+
+ :returns: whether the upload is cancellable
+ :rtype: bool
+ """
+ return self.status in ("WAITING", "READY", "RUNNING")
+
+ def cancel(self, status_callback=None):
+ """Cancel the upload. Sends a SIGINT to self.upload_pid.
+
+ :param status_callback: a function of the form callback(self)
+ :type status_callback: function
+ """
+ if not self.is_cancellable():
+ raise RuntimeError(f"Can't cancel, status is already {self.status}!")
+ if self.upload_pid:
+ os.kill(self.upload_pid, signal.SIGINT)
+ self.set_status("CANCELLED", status_callback)
+
+ def execute(self, status_callback=None):
+ """Execute the upload. Meant to be called from a dedicated process so
+ that the upload can be cancelled by sending a SIGINT to
+ self.upload_pid.
+
+ :param status_callback: a function of the form callback(self)
+ :type status_callback: function
+ """
+ if self.status != "READY":
+ raise RuntimeError("This upload is not ready!")
+ self.upload_pid = current_process().pid
+ self.set_status("RUNNING", status_callback)
+
+ logger = lambda e: self._log(e["stdout"], status_callback)
+
+ runner = ansible_run(
+ playbook=self.playbook_path,
+ extravars={
+ **self.settings,
+ "image_name": self.image_name,
+ "image_path": self.image_path,
+ },
+ event_handler=logger,
+ )
+ if runner.status == "successful":
+ self.set_status("FINISHED", status_callback)
+ else:
+ self.set_status("FAILED", status_callback)
diff --git a/src/pylorax/api/config.py b/src/pylorax/api/config.py
index 8f9a30de..1662ac4f 100644
--- a/src/pylorax/api/config.py
+++ b/src/pylorax/api/config.py
@@ -54,6 +54,11 @@ def configure(conf_file="/etc/lorax/composer.conf", root_dir="/", test_config=Fa
conf.add_section("users")
conf.set("users", "root", "1")
+ conf.add_section("upload")
+ conf.set("upload", "providers_dir", os.path.realpath(joinpaths(root_dir, "/usr/share/lorax/lifted/providers/")))
+ conf.set("upload", "queue_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/upload/queue")))
+ conf.set("upload", "settings_dir", os.path.realpath(joinpaths(root_dir, "/var/lib/lorax/upload/settings")))
+
# Enable all available repo files by default
conf.add_section("repos")
conf.set("repos", "use_system_repos", "1")
diff --git a/src/pylorax/api/errors.py b/src/pylorax/api/errors.py
index c4d1bd95..c4f45656 100644
--- a/src/pylorax/api/errors.py
+++ b/src/pylorax/api/errors.py
@@ -45,6 +45,9 @@ BUILD_MISSING_FILE = "BuildMissingFile"
# Returned from the API for all other errors from a /compose/* route.
COMPOSE_ERROR = "ComposeError"
+# Returned from the API for all errors from a /upload/* route.
+UPLOAD_ERROR = "UploadError" # TODO these errors should be more specific
+
# Returned from the API when invalid characters are used in a route path or in
# some identifier.
INVALID_CHARS = "InvalidChars"
diff --git a/src/pylorax/api/queue.py b/src/pylorax/api/queue.py
index 33470e4b..c5b7c5af 100644
--- a/src/pylorax/api/queue.py
+++ b/src/pylorax/api/queue.py
@@ -38,6 +38,8 @@ from pylorax.base import DataHolder
from pylorax.creator import run_creator
from pylorax.sysutils import joinpaths, read_tail
+from lifted.queue import create_upload, get_uploads, ready_upload, delete_upload
+
def check_queues(cfg):
"""Check to make sure the new and run queue symlinks are correct
@@ -93,7 +95,7 @@ def start_queue_monitor(cfg, uid, gid):
lib_dir = cfg.get("composer", "lib_dir")
share_dir = cfg.get("composer", "share_dir")
tmp = cfg.get("composer", "tmp")
- monitor_cfg = DataHolder(composer_dir=lib_dir, share_dir=share_dir, uid=uid, gid=gid, tmp=tmp)
+ monitor_cfg = DataHolder(cfg=cfg, composer_dir=lib_dir, share_dir=share_dir, uid=uid, gid=gid, tmp=tmp)
p = mp.Process(target=monitor, args=(monitor_cfg,))
p.daemon = True
p.start()
@@ -163,6 +165,11 @@ def monitor(cfg):
log.info("Finished building %s, results are in %s", dst, os.path.realpath(dst))
open(joinpaths(dst, "STATUS"), "w").write("FINISHED\n")
write_timestamp(dst, TS_FINISHED)
+
+ upload_cfg = cfg.cfg["upload"]
+ for upload in get_uploads(upload_cfg, uuid_get_uploads(cfg.cfg, uuids[0])):
+ log.info("Readying upload %s", upload.uuid)
+ uuid_ready_upload(cfg.cfg, uuids[0], upload.uuid)
except Exception:
import traceback
log.error("traceback: %s", traceback.format_exc())
@@ -298,7 +305,7 @@ def get_compose_type(results_dir):
raise RuntimeError("Cannot find ks template for build %s" % os.path.basename(results_dir))
return t[0]
-def compose_detail(results_dir):
+def compose_detail(cfg, results_dir):
"""Return details about the build.
:param results_dir: The directory containing the metadata and results for the build
@@ -338,6 +345,9 @@ def compose_detail(results_dir):
times = timestamp_dict(results_dir)
+ upload_uuids = uuid_get_uploads(cfg, build_id)
+ summaries = [upload.summary() for upload in get_uploads(cfg["upload"], upload_uuids)]
+
return {"id": build_id,
"queue_status": status,
"job_created": times.get(TS_CREATED),
@@ -346,7 +356,8 @@ def compose_detail(results_dir):
"compose_type": compose_type,
"blueprint": blueprint["name"],
"version": blueprint["version"],
- "image_size": image_size
+ "image_size": image_size,
+ "uploads": summaries,
}
def queue_status(cfg):
@@ -367,7 +378,7 @@ def queue_status(cfg):
new_details = []
for n in new_queue:
try:
- d = compose_detail(n)
+ d = compose_detail(cfg, n)
except IOError:
continue
new_details.append(d)
@@ -375,7 +386,7 @@ def queue_status(cfg):
run_details = []
for r in run_queue:
try:
- d = compose_detail(r)
+ d = compose_detail(cfg, r)
except IOError:
continue
run_details.append(d)
@@ -399,7 +410,7 @@ def uuid_status(cfg, uuid):
"""
uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid)
try:
- return compose_detail(uuid_dir)
+ return compose_detail(cfg, uuid_dir)
except IOError:
return None
@@ -430,11 +441,56 @@ def build_status(cfg, status_filter=None):
try:
status = open(joinpaths(build, "STATUS"), "r").read().strip()
if status in status_filter:
- results.append(compose_detail(build))
+ results.append(compose_detail(cfg, build))
except IOError:
pass
return results
+def _upload_list_path(cfg, uuid):
+ results_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid)
+ if not os.path.isdir(results_dir):
+ raise RuntimeError(f'"{uuid}" is not a valid build uuid!')
+ return joinpaths(results_dir, "UPLOADS")
+
+def uuid_schedule_upload(cfg, uuid, provider_name, image_name, settings):
+ status = uuid_status(cfg, uuid)
+ if status is None:
+ raise RuntimeError(f'"{uuid}" is not a valid build uuid!')
+
+ upload = create_upload(cfg["upload"], provider_name, image_name, settings)
+ uuid_add_upload(cfg, uuid, upload.uuid)
+ return upload.uuid
+
+def uuid_get_uploads(cfg, uuid):
+ try:
+ with open(_upload_list_path(cfg, uuid)) as uploads_file:
+ return frozenset(uploads_file.read().split())
+ except FileNotFoundError:
+ return frozenset()
+
+def uuid_add_upload(cfg, uuid, upload_uuid):
+ if upload_uuid not in uuid_get_uploads(cfg, uuid):
+ with open(_upload_list_path(cfg, uuid), "a") as uploads_file:
+ print(upload_uuid, file=uploads_file)
+ status = uuid_status(cfg, uuid)
+ if status and status["queue_status"] == "FINISHED":
+ uuid_ready_upload(cfg, uuid, upload_uuid)
+
+def uuid_remove_upload(cfg, uuid, upload_uuid):
+ uploads = uuid_get_uploads(cfg, uuid) - frozenset((upload_uuid,))
+ with open(_upload_list_path(cfg, uuid), "w") as uploads_file:
+ for upload in uploads:
+ print(upload, file=uploads_file)
+
+def uuid_ready_upload(cfg, uuid, upload_uuid):
+ status = uuid_status(cfg, uuid)
+ if not status:
+ raise RuntimeError(f"{uuid} is not a valid build id!")
+ if status["queue_status"] != "FINISHED":
+ raise RuntimeError(f"Build {uuid} is not finished!")
+ _, image_path = uuid_image(cfg, uuid)
+ ready_upload(cfg["upload"], upload_uuid, image_path)
+
def uuid_cancel(cfg, uuid):
"""Cancel a build and delete its results
@@ -510,6 +566,10 @@ def uuid_delete(cfg, uuid):
uuid_dir = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid)
if not uuid_dir or len(uuid_dir) < 10:
raise RuntimeError("Directory length is too short: %s" % uuid_dir)
+
+ for upload in get_uploads(cfg["upload"], uuid_get_uploads(cfg, uuid)):
+ delete_upload(cfg["upload"], upload.uuid)
+
shutil.rmtree(uuid_dir)
return True
@@ -554,13 +614,16 @@ def uuid_info(cfg, uuid):
raise RuntimeError("Missing deps.toml for %s" % uuid)
deps_dict = toml.loads(open(deps_path, "r").read())
- details = compose_detail(uuid_dir)
+ details = compose_detail(cfg, uuid_dir)
commit_path = joinpaths(uuid_dir, "COMMIT")
if not os.path.exists(commit_path):
raise RuntimeError("Missing commit hash for %s" % uuid)
commit_id = open(commit_path, "r").read().strip()
+ upload_uuids = uuid_get_uploads(cfg, uuid)
+ summaries = [upload.summary() for upload in get_uploads(cfg["upload"], upload_uuids)]
+
return {"id": uuid,
"config": cfg_dict,
"blueprint": frozen_dict,
@@ -568,7 +631,8 @@ def uuid_info(cfg, uuid):
"deps": deps_dict,
"compose_type": details["compose_type"],
"queue_status": details["queue_status"],
- "image_size": details["image_size"]
+ "image_size": details["image_size"],
+ "uploads": summaries,
}
def uuid_tar(cfg, uuid, metadata=False, image=False, logs=False):
diff --git a/src/pylorax/api/server.py b/src/pylorax/api/server.py
index b4dd5e98..9f3f9b2e 100644
--- a/src/pylorax/api/server.py
+++ b/src/pylorax/api/server.py
@@ -89,6 +89,10 @@ server.register_blueprint(v0_api, url_prefix="/api/v0/")
# Register the v1 API on /api/v1/
# Use v0 routes by default
-server.register_blueprint(v0_api, url_prefix="/api/v1/",
- skip_rules=["/projects/source/info/", "/projects/source/new"])
+skip_rules = [
+ "/compose",
+ "/projects/source/info/",
+ "/projects/source/new",
+]
+server.register_blueprint(v0_api, url_prefix="/api/v1/", skip_rules=skip_rules)
server.register_blueprint(v1_api, url_prefix="/api/v1/")
diff --git a/src/pylorax/api/v0.py b/src/pylorax/api/v0.py
index 31a2bce2..9d6825d2 100644
--- a/src/pylorax/api/v0.py
+++ b/src/pylorax/api/v0.py
@@ -58,7 +58,7 @@ from flask import current_app as api
from pylorax.sysutils import joinpaths
from pylorax.api.checkparams import checkparams
from pylorax.api.compose import start_build, compose_types
-from pylorax.api.errors import * # pylint: disable=wildcard-import
+from pylorax.api.errors import * # pylint: disable=wildcard-import,unused-wildcard-import
from pylorax.api.flask_blueprint import BlueprintSkip
from pylorax.api.projects import projects_list, projects_info, projects_depsolve
from pylorax.api.projects import modules_list, modules_info, ProjectsError, repo_to_source
diff --git a/src/pylorax/api/v1.py b/src/pylorax/api/v1.py
index bba96bf6..8c84482d 100644
--- a/src/pylorax/api/v1.py
+++ b/src/pylorax/api/v1.py
@@ -23,12 +23,20 @@ log = logging.getLogger("lorax-composer")
from flask import jsonify, request
from flask import current_app as api
+from lifted.queue import get_upload, reset_upload, cancel_upload, delete_upload
+from lifted.providers import list_providers, resolve_provider, load_profiles, validate_settings, save_settings
from pylorax.api.checkparams import checkparams
-from pylorax.api.errors import INVALID_CHARS, PROJECTS_ERROR, SYSTEM_SOURCE, UNKNOWN_SOURCE
+from pylorax.api.compose import start_build
+from pylorax.api.errors import BAD_COMPOSE_TYPE, BUILD_FAILED, INVALID_CHARS, MISSING_POST, PROJECTS_ERROR
+from pylorax.api.errors import SYSTEM_SOURCE, UNKNOWN_BLUEPRINT, UNKNOWN_SOURCE, UNKNOWN_UUID, UPLOAD_ERROR
from pylorax.api.flask_blueprint import BlueprintSkip
-from pylorax.api.projects import get_repo_sources, new_repo_source, repo_to_source
-from pylorax.api.regexes import VALID_API_STRING
+from pylorax.api.queue import uuid_status, uuid_schedule_upload, uuid_remove_upload
+from pylorax.api.projects import get_repo_sources, repo_to_source
+from pylorax.api.projects import new_repo_source
+from pylorax.api.regexes import VALID_API_STRING, VALID_BLUEPRINT_NAME
import pylorax.api.toml as toml
+from pylorax.api.utils import blueprint_exists
+
# Create the v1 routes Blueprint with skip_routes support
v1_api = BlueprintSkip("v1_routes", __name__)
@@ -166,3 +174,460 @@ def v1_projects_source_new():
return jsonify(status=False, errors=[{"id": PROJECTS_ERROR, "msg": str(e)}]), 400
return jsonify(status=True)
+
+@v1_api.route("/compose", methods=["POST"])
+def v1_compose_start():
+ """Start a compose
+
+ The body of the post should have these fields:
+ blueprint_name - The blueprint name from /blueprints/list/
+ compose_type - The type of output to create, from /compose/types
+ branch - Optional, defaults to master, selects the git branch to use for the blueprint.
+
+ **POST /api/v0/compose**
+
+ Start a compose. The content type should be 'application/json' and the body of the POST
+ should look like this. The "upload" object is optional.
+
+ Example::
+
+ {
+ "blueprint_name": "http-server",
+ "compose_type": "tar",
+ "branch": "master",
+ "upload": {
+ "image_name": "My Image",
+ "provider": "azure",
+ "settings": {
+ "resource_group": "SOMEBODY",
+ "storage_account_name": "ONCE",
+ "storage_container": "TOLD",
+ "location": "ME",
+ "subscription_id": "THE",
+ "client_id": "WORLD",
+ "secret": "IS",
+ "tenant": "GONNA"
+ }
+ }
+ }
+
+ Pass it the name of the blueprint, the type of output (from
+ '/api/v0/compose/types'), and the blueprint branch to use. 'branch' is
+ optional and will default to master. It will create a new build and add
+ it to the queue. It returns the build uuid and a status if it succeeds.
+ If an "upload" is given, it will schedule an upload to run when the build
+ finishes.
+
+ Example::
+
+ {
+ "build_id": "e6fa6db4-9c81-4b70-870f-a697ca405cdf",
+ "status": true
+ }
+ """
+ # Passing ?test=1 will generate a fake FAILED compose.
+ # Passing ?test=2 will generate a fake FINISHED compose.
+ try:
+ test_mode = int(request.args.get("test", "0"))
+ except ValueError:
+ test_mode = 0
+
+ compose = request.get_json(cache=False)
+
+ errors = []
+ if not compose:
+ return jsonify(status=False, errors=[{"id": MISSING_POST, "msg": "Missing POST body"}]), 400
+
+ if "blueprint_name" not in compose:
+ errors.append({"id": UNKNOWN_BLUEPRINT, "msg": "No 'blueprint_name' in the JSON request"})
+ else:
+ blueprint_name = compose["blueprint_name"]
+
+ if "branch" not in compose or not compose["branch"]:
+ branch = "master"
+ else:
+ branch = compose["branch"]
+
+ if "compose_type" not in compose:
+ errors.append({"id": BAD_COMPOSE_TYPE, "msg": "No 'compose_type' in the JSON request"})
+ else:
+ compose_type = compose["compose_type"]
+
+ if VALID_BLUEPRINT_NAME.match(blueprint_name) is None:
+ errors.append({"id": INVALID_CHARS, "msg": "Invalid characters in API path"})
+
+ if not blueprint_exists(api, branch, blueprint_name):
+ errors.append({"id": UNKNOWN_BLUEPRINT, "msg": "Unknown blueprint name: %s" % blueprint_name})
+
+ if "upload" in compose:
+ try:
+ image_name = compose["upload"]["image_name"]
+ provider_name = compose["upload"]["provider"]
+ settings = compose["upload"]["settings"]
+ except KeyError as e:
+ errors.append({"id": UPLOAD_ERROR, "msg": f'Missing parameter {str(e)}!'})
+ try:
+ provider = resolve_provider(api.config["COMPOSER_CFG"]["upload"], provider_name)
+ if "supported_types" in provider and compose_type not in provider["supported_types"]:
+ raise RuntimeError(f'Type "{compose_type}" is not supported by provider "{provider_name}"!')
+ validate_settings(api.config["COMPOSER_CFG"]["upload"], provider_name, settings, image_name)
+ except Exception as e:
+ errors.append({"id": UPLOAD_ERROR, "msg": str(e)})
+
+ if errors:
+ return jsonify(status=False, errors=errors), 400
+
+ try:
+ build_id = start_build(api.config["COMPOSER_CFG"], api.config["DNFLOCK"], api.config["GITLOCK"],
+ branch, blueprint_name, compose_type, test_mode)
+ except Exception as e:
+ if "Invalid compose type" in str(e):
+ return jsonify(status=False, errors=[{"id": BAD_COMPOSE_TYPE, "msg": str(e)}]), 400
+ else:
+ return jsonify(status=False, errors=[{"id": BUILD_FAILED, "msg": str(e)}]), 400
+
+ if "upload" in compose:
+ upload_uuid = uuid_schedule_upload(
+ api.config["COMPOSER_CFG"],
+ build_id,
+ provider_name,
+ image_name,
+ settings
+ )
+
+ return jsonify(status=True, build_id=build_id)
+
+@v1_api.route("/compose/uploads/schedule", defaults={'compose_uuid': ""}, methods=["POST"])
+@v1_api.route("/compose/uploads/schedule/", methods=["POST"])
+@checkparams([("compose_uuid", "", "no compose UUID given")])
+def v1_compose_uploads_schedule(compose_uuid):
+ """Schedule an upload of a compose to a given cloud provider
+
+ **POST /api/v1/uploads/schedule/**
+
+ Example request::
+
+ {
+ "image_name": "My Image",
+ "provider": "azure",
+ "settings": {
+ "resource_group": "SOMEBODY",
+ "storage_account_name": "ONCE",
+ "storage_container": "TOLD",
+ "location": "ME",
+ "subscription_id": "THE",
+ "client_id": "WORLD",
+ "secret": "IS",
+ "tenant": "GONNA"
+ }
+ }
+
+ Example response::
+
+ {
+ "status": true,
+ "upload_uuid": "572eb0d0-5348-4600-9666-14526ba628bb"
+ }
+ """
+ if VALID_API_STRING.match(compose_uuid) is None:
+ error = {"id": INVALID_CHARS, "msg": "Invalid characters in API path"}
+ return jsonify(status=False, errors=[error]), 400
+
+ parsed = request.get_json(cache=False)
+ if not parsed:
+ return jsonify(status=False, errors=[{"id": MISSING_POST, "msg": "Missing POST body"}]), 400
+
+ try:
+ image_name = parsed["image_name"]
+ provider_name = parsed["provider"]
+ settings = parsed["settings"]
+ except KeyError as e:
+ error = {"id": UPLOAD_ERROR, "msg": f'Missing parameter {str(e)}!'}
+ return jsonify(status=False, errors=[error]), 400
+ try:
+ compose_type = uuid_status(api.config["COMPOSER_CFG"], compose_uuid)["compose_type"]
+ provider = resolve_provider(api.config["COMPOSER_CFG"]["upload"], provider_name)
+ if "supported_types" in provider and compose_type not in provider["supported_types"]:
+ raise RuntimeError(
+ f'Type "{compose_type}" is not supported by provider "{provider_name}"!'
+ )
+ except Exception as e:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(e)}]), 400
+
+ try:
+ upload_uuid = uuid_schedule_upload(
+ api.config["COMPOSER_CFG"],
+ compose_uuid,
+ provider_name,
+ image_name,
+ settings
+ )
+ except RuntimeError as e:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(e)}]), 400
+ return jsonify(status=True, upload_uuid=upload_uuid)
+
+@v1_api.route("/compose/uploads/delete", defaults={"compose_uuid": "", "upload_uuid": ""}, methods=["DELETE"])
+@v1_api.route("/compose/uploads/delete//", methods=["DELETE"])
+@checkparams([("compose_uuid", "", "no compose UUID given"), ("upload_uuid", "", "no upload UUID given")])
+def v1_compose_uploads_delete(compose_uuid, upload_uuid):
+ """Delete an upload and disassociate it from its compose
+
+ **DELETE /api/v1/uploads/delete//**
+
+ Example response::
+
+ {
+ "status": true,
+ "upload_uuid": "572eb0d0-5348-4600-9666-14526ba628bb"
+ }
+ """
+ if None in (VALID_API_STRING.match(compose_uuid), VALID_API_STRING.match(upload_uuid)):
+ error = {"id": INVALID_CHARS, "msg": "Invalid characters in API path"}
+ return jsonify(status=False, errors=[error]), 400
+
+ if not uuid_status(api.config["COMPOSER_CFG"], compose_uuid):
+ error = {"id": UNKNOWN_UUID, "msg": "%s is not a valid build uuid" % compose_uuid}
+ return jsonify(status=False, errors=[error]), 400
+ uuid_remove_upload(api.config["COMPOSER_CFG"], compose_uuid, upload_uuid)
+ try:
+ delete_upload(api.config["COMPOSER_CFG"]["upload"], upload_uuid)
+ except RuntimeError as error:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(error)}])
+ return jsonify(status=True, upload_uuid=upload_uuid)
+
+@v1_api.route("/upload/info", defaults={"uuid": ""})
+@v1_api.route("/upload/info/")
+@checkparams([("uuid", "", "no UUID given")])
+def v1_upload_info(uuid):
+ """Returns information about a given upload
+
+ **GET /api/v1/upload/info/**
+
+ Example response::
+
+ {
+ "status": true,
+ "upload": {
+ "creation_time": 1565620940.069004,
+ "image_name": "My Image",
+ "image_path": "/var/lib/lorax/composer/results/b6218e8f-0fa2-48ec-9394-f5c2918544c4/disk.vhd",
+ "provider_name": "azure",
+ "settings": {
+ "resource_group": "SOMEBODY",
+ "storage_account_name": "ONCE",
+ "storage_container": "TOLD",
+ "location": "ME",
+ "subscription_id": "THE",
+ "client_id": "WORLD",
+ "secret": "IS",
+ "tenant": "GONNA"
+ },
+ "status": "FAILED",
+ "uuid": "b637c411-9d9d-4279-b067-6c8d38e3b211"
+ }
+ }
+ """
+ if VALID_API_STRING.match(uuid) is None:
+ return jsonify(status=False, errors=[{"id": INVALID_CHARS, "msg": "Invalid characters in API path"}]), 400
+
+ try:
+ upload = get_upload(api.config["COMPOSER_CFG"]["upload"], uuid).summary()
+ except RuntimeError as error:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(error)}])
+ return jsonify(status=True, upload=upload)
+
+@v1_api.route("/upload/log", defaults={"uuid": ""})
+@v1_api.route("/upload/log/")
+@checkparams([("uuid", "", "no UUID given")])
+def v1_upload_log(uuid):
+ """Returns an upload's log
+
+ **GET /api/v1/upload/log/**
+
+ Example response::
+
+ {
+ "status": true,
+ "log": "\n __________________\r\n< PLAY [localhost] >..."
+ }
+ """
+ if VALID_API_STRING.match(uuid) is None:
+ error = {"id": INVALID_CHARS, "msg": "Invalid characters in API path"}
+ return jsonify(status=False, errors=[error]), 400
+
+ try:
+ upload = get_upload(api.config["COMPOSER_CFG"]["upload"], uuid)
+ except RuntimeError as error:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(error)}])
+ return jsonify(status=True, log=upload.upload_log)
+
+@v1_api.route("/upload/reset", defaults={"uuid": ""}, methods=["POST"])
+@v1_api.route("/upload/reset/", methods=["POST"])
+@checkparams([("uuid", "", "no UUID given")])
+def v1_upload_reset(uuid):
+ """Reset an upload so it can be attempted again
+
+ **POST /api/v1/upload/reset/**
+
+ Optionally pass in a new image name and/or new settings.
+
+ Example request::
+
+ {
+ "image_name": "My renamed image",
+ "settings": {
+ "resource_group": "ROLL",
+ "storage_account_name": "ME",
+ "storage_container": "I",
+ "location": "AIN'T",
+ "subscription_id": "THE",
+ "client_id": "SHARPEST",
+ "secret": "TOOL",
+ "tenant": "IN"
+ }
+ }
+
+ Example response::
+
+ {
+ "status": true,
+ "uuid": "c75d5d62-9d26-42fc-a8ef-18bb14679fc7"
+ }
+ """
+ if VALID_API_STRING.match(uuid) is None:
+ error = {"id": INVALID_CHARS, "msg": "Invalid characters in API path"}
+ return jsonify(status=False, errors=[error]), 400
+
+ parsed = request.get_json(cache=False)
+ image_name = parsed.get("image_name") if parsed else None
+ settings = parsed.get("settings") if parsed else None
+
+ try:
+ reset_upload(api.config["COMPOSER_CFG"]["upload"], uuid, image_name, settings)
+ except RuntimeError as error:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(error)}])
+ return jsonify(status=True, uuid=uuid)
+
+@v1_api.route("/upload/cancel", defaults={"uuid": ""}, methods=["DELETE"])
+@v1_api.route("/upload/cancel/", methods=["DELETE"])
+@checkparams([("uuid", "", "no UUID given")])
+def v1_upload_cancel(uuid):
+ """Cancel an upload that is either queued or in progress
+
+ **DELETE /api/v1/uploads/delete//**
+
+ Example response::
+
+ {
+ "status": true,
+ "uuid": "037a3d56-b421-43e9-9935-c98350c89996"
+ }
+ """
+ if VALID_API_STRING.match(uuid) is None:
+ error = {"id": INVALID_CHARS, "msg": "Invalid characters in API path"}
+ return jsonify(status=False, errors=[error]), 400
+
+ try:
+ cancel_upload(api.config["COMPOSER_CFG"]["upload"], uuid)
+ except RuntimeError as error:
+ return jsonify(status=False, errors=[{"id": UPLOAD_ERROR, "msg": str(error)}])
+ return jsonify(status=True, uuid=uuid)
+
+@v1_api.route("/upload/providers")
+def v1_upload_providers():
+ """Return the information about all upload providers, including their
+ display names, expected settings, and saved profiles. Refer to the
+ `resolve_provider` function.
+
+ **GET /api/v1/upload/providers**
+
+ Example response::
+
+ {
+ "providers": {
+ "azure": {
+ "display": "Azure",
+ "profiles": {
+ "default": {
+ "client_id": "example",
+ ...
+ }
+ },
+ "settings-info": {
+ "client_id": {
+ "display": "Client ID",
+ "placeholder": "",
+ "regex": "",
+ "type": "string"
+ },
+ ...
+ },
+ "supported_types": ["vhd"]
+ },
+ ...
+ }
+ }
+ """
+
+ ucfg = api.config["COMPOSER_CFG"]["upload"]
+
+ provider_names = list_providers(ucfg)
+
+ def get_provider_info(provider_name):
+ provider = resolve_provider(ucfg, provider_name)
+ provider["profiles"] = load_profiles(ucfg, provider_name)
+ return provider
+
+ providers = {provider_name: get_provider_info(provider_name)
+ for provider_name in provider_names}
+ return jsonify(status=True, providers=providers)
+
+@v1_api.route("/upload/providers/save", methods=["POST"])
+def v1_providers_save():
+ """Save provider settings as a profile for later use
+
+ **POST /api/v1/upload/providers/save**
+
+ Example request::
+
+ {
+ "provider": "azure",
+ "profile": "my-profile",
+ "settings": {
+ "resource_group": "SOMEBODY",
+ "storage_account_name": "ONCE",
+ "storage_container": "TOLD",
+ "location": "ME",
+ "subscription_id": "THE",
+ "client_id": "WORLD",
+ "secret": "IS",
+ "tenant": "GONNA"
+ }
+ }
+
+ Saving to an existing profile will overwrite it.
+
+ Example response::
+
+ {
+ "status": true
+ }
+ """
+ parsed = request.get_json(cache=False)
+
+ if parsed is None:
+ return jsonify(status=False, errors=[{"id": MISSING_POST, "msg": "Missing POST body"}]), 400
+
+ try:
+ provider_name = parsed["provider"]
+ profile = parsed["profile"]
+ settings = parsed["settings"]
+ except KeyError as e:
+ error = {"id": UPLOAD_ERROR, "msg": f'Missing parameter {str(e)}!'}
+ return jsonify(status=False, errors=[error]), 400
+ try:
+ save_settings(api.config["COMPOSER_CFG"]["upload"], provider_name, profile, settings)
+ except Exception as e:
+ error = {"id": UPLOAD_ERROR, "msg": str(e)}
+ return jsonify(status=False, errors=[error])
+ return jsonify(status=True)
diff --git a/src/sbin/lorax-composer b/src/sbin/lorax-composer
index 581c2282..9fff5781 100755
--- a/src/sbin/lorax-composer
+++ b/src/sbin/lorax-composer
@@ -23,6 +23,7 @@ program_log = logging.getLogger("program")
pylorax_log = logging.getLogger("pylorax")
server_log = logging.getLogger("server")
dnf_log = logging.getLogger("dnf")
+lifted_log = logging.getLogger("lifted")
import grp
import os
@@ -43,12 +44,15 @@ from pylorax.api.queue import start_queue_monitor
from pylorax.api.recipes import open_or_create_repo, commit_recipe_directory
from pylorax.api.server import server, GitLock
+from lifted.queue import start_upload_monitor
+
VERSION = "{0}-{1}".format(os.path.basename(sys.argv[0]), vernum)
def setup_logging(logfile):
# Setup logging to console and to logfile
log.setLevel(logging.DEBUG)
pylorax_log.setLevel(logging.DEBUG)
+ lifted_log.setLevel(logging.DEBUG)
sh = logging.StreamHandler()
sh.setLevel(logging.INFO)
@@ -56,6 +60,7 @@ def setup_logging(logfile):
sh.setFormatter(fmt)
log.addHandler(sh)
pylorax_log.addHandler(sh)
+ lifted_log.addHandler(sh)
fh = logging.FileHandler(filename=logfile)
fh.setLevel(logging.DEBUG)
@@ -63,6 +68,7 @@ def setup_logging(logfile):
fh.setFormatter(fmt)
log.addHandler(fh)
pylorax_log.addHandler(fh)
+ lifted_log.addHandler(fh)
# External program output log
program_log.setLevel(logging.DEBUG)
@@ -244,6 +250,8 @@ if __name__ == '__main__':
start_queue_monitor(server.config["COMPOSER_CFG"], uid, gid)
+ start_upload_monitor(server.config["COMPOSER_CFG"]["upload"])
+
# Change user and group on the main process. Note that this still happens even if
# --user and --group were passed in, but changing to the same user should be fine.
os.setgid(gid)