467 lines
35 KiB
HTML
467 lines
35 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
<title>lifted.queue — Lorax 35.0 documentation</title>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<script type="text/javascript" src="../../_static/js/modernizr.min.js"></script>
|
|
|
|
|
|
<script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
|
|
<script src="../../_static/jquery.js"></script>
|
|
<script src="../../_static/underscore.js"></script>
|
|
<script src="../../_static/doctools.js"></script>
|
|
<script src="../../_static/language_data.js"></script>
|
|
|
|
<script type="text/javascript" src="../../_static/js/theme.js"></script>
|
|
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
|
|
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
|
|
<link rel="index" title="Index" href="../../genindex.html" />
|
|
<link rel="search" title="Search" href="../../search.html" />
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav">
|
|
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
|
<div class="wy-side-scroll">
|
|
<div class="wy-side-nav-search" >
|
|
|
|
|
|
|
|
<a href="../../index.html" class="icon icon-home"> Lorax
|
|
|
|
|
|
|
|
</a>
|
|
|
|
|
|
|
|
|
|
<div class="version">
|
|
35.0
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<div role="search">
|
|
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<ul>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../intro.html">Introduction to Lorax</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../intro.html#before-lorax">Before Lorax</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../lorax.html">Lorax</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../livemedia-creator.html">livemedia-creator</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../composer-cli.html">composer-cli</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../mkksiso.html">mkksiso</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../product-images.html">Product and Updates Images</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="../../modules.html">src</a></li>
|
|
</ul>
|
|
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" aria-label="top navigation">
|
|
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="../../index.html">Lorax</a>
|
|
|
|
</nav>
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
|
|
<div class="rst-content">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
|
|
<ul class="wy-breadcrumbs">
|
|
|
|
<li><a href="../../index.html">Docs</a> »</li>
|
|
|
|
<li><a href="../index.html">Module code</a> »</li>
|
|
|
|
<li>lifted.queue</li>
|
|
|
|
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
|
|
|
|
<hr/>
|
|
</div>
|
|
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
|
<div itemprop="articleBody">
|
|
|
|
<h1>Source code for lifted.queue</h1><div class="highlight"><pre>
|
|
<span></span><span class="c1">#</span>
|
|
<span class="c1"># Copyright (C) 2019 Red Hat, Inc.</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># This program is free software; you can redistribute it and/or modify</span>
|
|
<span class="c1"># it under the terms of the GNU General Public License as published by</span>
|
|
<span class="c1"># the Free Software Foundation; either version 2 of the License, or</span>
|
|
<span class="c1"># (at your option) any later version.</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># This program is distributed in the hope that it will be useful,</span>
|
|
<span class="c1"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
|
|
<span class="c1"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
|
|
<span class="c1"># GNU General Public License for more details.</span>
|
|
<span class="c1">#</span>
|
|
<span class="c1"># You should have received a copy of the GNU General Public License</span>
|
|
<span class="c1"># along with this program. If not, see <http://www.gnu.org/licenses/>.</span>
|
|
<span class="c1">#</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="n">partial</span>
|
|
<span class="kn">from</span> <span class="nn">glob</span> <span class="kn">import</span> <span class="n">glob</span>
|
|
<span class="kn">import</span> <span class="nn">logging</span>
|
|
<span class="kn">import</span> <span class="nn">multiprocessing</span>
|
|
|
|
<span class="c1"># We use a multiprocessing Pool for uploads so that we can cancel them with a</span>
|
|
<span class="c1"># simple SIGINT, which should bubble down to subprocesses.</span>
|
|
<span class="kn">from</span> <span class="nn">multiprocessing</span> <span class="kn">import</span> <span class="n">Pool</span>
|
|
|
|
<span class="c1"># multiprocessing.dummy is to threads as multiprocessing is to processes.</span>
|
|
<span class="c1"># Since daemonic processes can't have children, we use a thread to monitor the</span>
|
|
<span class="c1"># upload pool.</span>
|
|
<span class="kn">from</span> <span class="nn">multiprocessing.dummy</span> <span class="kn">import</span> <span class="n">Process</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">operator</span> <span class="kn">import</span> <span class="n">attrgetter</span>
|
|
<span class="kn">import</span> <span class="nn">os</span>
|
|
<span class="kn">import</span> <span class="nn">stat</span>
|
|
<span class="kn">import</span> <span class="nn">time</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">pylorax.api.toml</span> <span class="k">as</span> <span class="nn">toml</span>
|
|
|
|
<span class="kn">from</span> <span class="nn">lifted.upload</span> <span class="kn">import</span> <span class="n">Upload</span>
|
|
<span class="kn">from</span> <span class="nn">lifted.providers</span> <span class="kn">import</span> <span class="n">resolve_playbook_path</span><span class="p">,</span> <span class="n">validate_settings</span>
|
|
|
|
<span class="c1"># the maximum number of simultaneous uploads</span>
|
|
<span class="n">SIMULTANEOUS_UPLOADS</span> <span class="o">=</span> <span class="mi">1</span>
|
|
|
|
<span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">"lifted"</span><span class="p">)</span>
|
|
<span class="n">multiprocessing</span><span class="o">.</span><span class="n">log_to_stderr</span><span class="p">()</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_get_queue_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="n">path</span> <span class="o">=</span> <span class="n">ucfg</span><span class="p">[</span><span class="s2">"queue_dir"</span><span class="p">]</span>
|
|
|
|
<span class="c1"># create the upload_queue directory if it doesn't exist</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
|
|
<span class="k">return</span> <span class="n">path</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_get_upload_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> <span class="n">write</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="c1"># Make sure no path elements are present</span>
|
|
<span class="n">uuid</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">uuid</span><span class="p">)</span>
|
|
|
|
<span class="n">path</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_get_queue_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">),</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="n">uuid</span><span class="si">}</span><span class="s2">.toml"</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">write</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
|
<span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s2">"a"</span><span class="p">)</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
|
<span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">):</span>
|
|
<span class="c1"># make sure uploads aren't readable by others, as they will contain</span>
|
|
<span class="c1"># sensitive credentials</span>
|
|
<span class="n">current</span> <span class="o">=</span> <span class="n">stat</span><span class="o">.</span><span class="n">S_IMODE</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">lstat</span><span class="p">(</span><span class="n">path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mode</span><span class="p">)</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">chmod</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">current</span> <span class="o">&</span> <span class="o">~</span><span class="n">stat</span><span class="o">.</span><span class="n">S_IROTH</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">path</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_list_upload_uuids</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="n">paths</span> <span class="o">=</span> <span class="n">glob</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">_get_queue_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">),</span> <span class="s2">"*"</span><span class="p">))</span>
|
|
<span class="k">return</span> <span class="p">[</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">basename</span><span class="p">(</span><span class="n">path</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">path</span> <span class="ow">in</span> <span class="n">paths</span><span class="p">]</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_write_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">upload</span><span class="p">):</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">_get_upload_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">upload</span><span class="o">.</span><span class="n">uuid</span><span class="p">,</span> <span class="n">write</span><span class="o">=</span><span class="kc">True</span><span class="p">),</span> <span class="s2">"w"</span><span class="p">)</span> <span class="k">as</span> <span class="n">upload_file</span><span class="p">:</span>
|
|
<span class="n">toml</span><span class="o">.</span><span class="n">dump</span><span class="p">(</span><span class="n">upload</span><span class="o">.</span><span class="n">serializable</span><span class="p">(),</span> <span class="n">upload_file</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="n">partial</span><span class="p">(</span><span class="n">_write_upload</span><span class="p">,</span> <span class="n">ucfg</span><span class="p">)</span>
|
|
|
|
|
|
<div class="viewcode-block" id="get_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.get_upload">[docs]</a><span class="k">def</span> <span class="nf">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> <span class="n">ignore_missing</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">ignore_corrupt</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
|
<span class="sd">"""Get an Upload object by UUID</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :param uuid: UUID of the upload to get</span>
|
|
<span class="sd"> :type uuid: str</span>
|
|
<span class="sd"> :param ignore_missing: if True, don't raise a RuntimeError when the specified upload is missing, instead just return None</span>
|
|
<span class="sd"> :type ignore_missing: bool</span>
|
|
<span class="sd"> :param ignore_corrupt: if True, don't raise a RuntimeError when the specified upload could not be deserialized, instead just return None</span>
|
|
<span class="sd"> :type ignore_corrupt: bool</span>
|
|
<span class="sd"> :returns: the upload object or None</span>
|
|
<span class="sd"> :rtype: Upload or None</span>
|
|
<span class="sd"> :raises: RuntimeError</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">try</span><span class="p">:</span>
|
|
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">_get_upload_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">),</span> <span class="s2">"r"</span><span class="p">)</span> <span class="k">as</span> <span class="n">upload_file</span><span class="p">:</span>
|
|
<span class="k">return</span> <span class="n">Upload</span><span class="p">(</span><span class="o">**</span><span class="n">toml</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">upload_file</span><span class="p">))</span>
|
|
<span class="k">except</span> <span class="ne">FileNotFoundError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_missing</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not find upload </span><span class="si">{</span><span class="n">uuid</span><span class="si">}</span><span class="s2">!"</span><span class="p">)</span> <span class="kn">from</span> <span class="nn">error</span>
|
|
<span class="k">except</span> <span class="n">toml</span><span class="o">.</span><span class="n">TomlError</span> <span class="k">as</span> <span class="n">error</span><span class="p">:</span>
|
|
<span class="k">if</span> <span class="ow">not</span> <span class="n">ignore_corrupt</span><span class="p">:</span>
|
|
<span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Could not parse upload </span><span class="si">{</span><span class="n">uuid</span><span class="si">}</span><span class="s2">!"</span><span class="p">)</span> <span class="kn">from</span> <span class="nn">error</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="get_uploads"><a class="viewcode-back" href="../../lifted.html#lifted.queue.get_uploads">[docs]</a><span class="k">def</span> <span class="nf">get_uploads</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuids</span><span class="p">):</span>
|
|
<span class="sd">"""Gets a list of Upload objects from a list of upload UUIDs, ignoring</span>
|
|
<span class="sd"> missing or corrupt uploads</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :param uuids: list of upload UUIDs to get</span>
|
|
<span class="sd"> :type uuids: list of str</span>
|
|
<span class="sd"> :returns: a list of the uploads that were successfully deserialized</span>
|
|
<span class="sd"> :rtype: list of Upload</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">uploads</span> <span class="o">=</span> <span class="p">(</span>
|
|
<span class="n">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> <span class="n">ignore_missing</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">ignore_corrupt</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">uuid</span> <span class="ow">in</span> <span class="n">uuids</span>
|
|
<span class="p">)</span>
|
|
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">filter</span><span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="n">uploads</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="get_all_uploads"><a class="viewcode-back" href="../../lifted.html#lifted.queue.get_all_uploads">[docs]</a><span class="k">def</span> <span class="nf">get_all_uploads</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="sd">"""Get a list of all stored Upload objects</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :returns: a list of all stored upload objects</span>
|
|
<span class="sd"> :rtype: list of Upload</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">return</span> <span class="n">get_uploads</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">_list_upload_uuids</span><span class="p">(</span><span class="n">ucfg</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="create_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.create_upload">[docs]</a><span class="k">def</span> <span class="nf">create_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">provider_name</span><span class="p">,</span> <span class="n">image_name</span><span class="p">,</span> <span class="n">settings</span><span class="p">):</span>
|
|
<span class="sd">"""Creates a new upload</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :param provider_name: the name of the cloud provider to upload to, e.g. "azure"</span>
|
|
<span class="sd"> :type provider_name: str</span>
|
|
<span class="sd"> :param image_name: what to name the image in the cloud</span>
|
|
<span class="sd"> :type image_name: str</span>
|
|
<span class="sd"> :param settings: settings to pass to the upload, specific to the cloud provider</span>
|
|
<span class="sd"> :type settings: dict</span>
|
|
<span class="sd"> :returns: the created upload object</span>
|
|
<span class="sd"> :rtype: Upload</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">validate_settings</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">provider_name</span><span class="p">,</span> <span class="n">settings</span><span class="p">,</span> <span class="n">image_name</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">Upload</span><span class="p">(</span>
|
|
<span class="n">provider_name</span><span class="o">=</span><span class="n">provider_name</span><span class="p">,</span>
|
|
<span class="n">playbook_path</span><span class="o">=</span><span class="n">resolve_playbook_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">provider_name</span><span class="p">),</span>
|
|
<span class="n">image_name</span><span class="o">=</span><span class="n">image_name</span><span class="p">,</span>
|
|
<span class="n">settings</span><span class="o">=</span><span class="n">settings</span><span class="p">,</span>
|
|
<span class="n">status_callback</span><span class="o">=</span><span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">),</span>
|
|
<span class="p">)</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="ready_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.ready_upload">[docs]</a><span class="k">def</span> <span class="nf">ready_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> <span class="n">image_path</span><span class="p">):</span>
|
|
<span class="sd">"""Pass an image_path to an upload and mark it ready to execute</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :param uuid: the UUID of the upload to mark ready</span>
|
|
<span class="sd"> :type uuid: str</span>
|
|
<span class="sd"> :param image_path: the path of the image to pass to the upload</span>
|
|
<span class="sd"> :type image_path: str</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">)</span><span class="o">.</span><span class="n">ready</span><span class="p">(</span><span class="n">image_path</span><span class="p">,</span> <span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="reset_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.reset_upload">[docs]</a><span class="k">def</span> <span class="nf">reset_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">,</span> <span class="n">new_image_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">new_settings</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
|
|
<span class="sd">"""Reset an upload so it can be attempted again</span>
|
|
|
|
<span class="sd"> :param ucfg: upload config</span>
|
|
<span class="sd"> :type ucfg: object</span>
|
|
<span class="sd"> :param uuid: the UUID of the upload to reset</span>
|
|
<span class="sd"> :type uuid: str</span>
|
|
<span class="sd"> :param new_image_name: optionally update the upload's image_name</span>
|
|
<span class="sd"> :type new_image_name: str</span>
|
|
<span class="sd"> :param new_settings: optionally update the upload's settings</span>
|
|
<span class="sd"> :type new_settings: dict</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">upload</span> <span class="o">=</span> <span class="n">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">)</span>
|
|
<span class="n">validate_settings</span><span class="p">(</span>
|
|
<span class="n">ucfg</span><span class="p">,</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">provider_name</span><span class="p">,</span>
|
|
<span class="n">new_settings</span> <span class="ow">or</span> <span class="n">upload</span><span class="o">.</span><span class="n">settings</span><span class="p">,</span>
|
|
<span class="n">new_image_name</span> <span class="ow">or</span> <span class="n">upload</span><span class="o">.</span><span class="n">image_name</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">new_image_name</span><span class="p">:</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">image_name</span> <span class="o">=</span> <span class="n">new_image_name</span>
|
|
<span class="k">if</span> <span class="n">new_settings</span><span class="p">:</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">settings</span> <span class="o">=</span> <span class="n">new_settings</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">reset</span><span class="p">(</span><span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="cancel_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.cancel_upload">[docs]</a><span class="k">def</span> <span class="nf">cancel_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">):</span>
|
|
<span class="sd">"""Cancel an upload</span>
|
|
|
|
<span class="sd"> :param ucfg: the compose config</span>
|
|
<span class="sd"> :type ucfg: ComposerConfig</span>
|
|
<span class="sd"> :param uuid: the UUID of the upload to cancel</span>
|
|
<span class="sd"> :type uuid: str</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">)</span><span class="o">.</span><span class="n">cancel</span><span class="p">(</span><span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="delete_upload"><a class="viewcode-back" href="../../lifted.html#lifted.queue.delete_upload">[docs]</a><span class="k">def</span> <span class="nf">delete_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">):</span>
|
|
<span class="sd">"""Delete an upload</span>
|
|
|
|
<span class="sd"> :param ucfg: the compose config</span>
|
|
<span class="sd"> :type ucfg: ComposerConfig</span>
|
|
<span class="sd"> :param uuid: the UUID of the upload to delete</span>
|
|
<span class="sd"> :type uuid: str</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">upload</span> <span class="o">=</span> <span class="n">get_upload</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">)</span>
|
|
<span class="k">if</span> <span class="n">upload</span> <span class="ow">and</span> <span class="n">upload</span><span class="o">.</span><span class="n">is_cancellable</span><span class="p">():</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">cancel</span><span class="p">()</span>
|
|
<span class="n">os</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">_get_upload_path</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,</span> <span class="n">uuid</span><span class="p">))</span></div>
|
|
|
|
|
|
<div class="viewcode-block" id="start_upload_monitor"><a class="viewcode-back" href="../../lifted.html#lifted.queue.start_upload_monitor">[docs]</a><span class="k">def</span> <span class="nf">start_upload_monitor</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="sd">"""Start a thread that manages the upload queue</span>
|
|
|
|
<span class="sd"> :param ucfg: the compose config</span>
|
|
<span class="sd"> :type ucfg: ComposerConfig</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">process</span> <span class="o">=</span> <span class="n">Process</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">_monitor</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">ucfg</span><span class="p">,))</span>
|
|
<span class="n">process</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
|
|
<span class="n">process</span><span class="o">.</span><span class="n">start</span><span class="p">()</span></div>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">_monitor</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Started upload monitor."</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">upload</span> <span class="ow">in</span> <span class="n">get_all_uploads</span><span class="p">(</span><span class="n">ucfg</span><span class="p">):</span>
|
|
<span class="c1"># Set abandoned uploads to FAILED</span>
|
|
<span class="k">if</span> <span class="n">upload</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s2">"RUNNING"</span><span class="p">:</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">set_status</span><span class="p">(</span><span class="s2">"FAILED"</span><span class="p">,</span> <span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">))</span>
|
|
<span class="n">pool</span> <span class="o">=</span> <span class="n">Pool</span><span class="p">(</span><span class="n">processes</span><span class="o">=</span><span class="n">SIMULTANEOUS_UPLOADS</span><span class="p">)</span>
|
|
<span class="n">pool_uuids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
|
|
|
|
<span class="k">def</span> <span class="nf">remover</span><span class="p">(</span><span class="n">uuid</span><span class="p">):</span>
|
|
<span class="k">return</span> <span class="k">lambda</span> <span class="n">_</span><span class="p">:</span> <span class="n">pool_uuids</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">uuid</span><span class="p">)</span>
|
|
|
|
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
|
|
<span class="c1"># Every second, scoop up READY uploads from the filesystem and throw</span>
|
|
<span class="c1"># them in the pool</span>
|
|
<span class="n">all_uploads</span> <span class="o">=</span> <span class="n">get_all_uploads</span><span class="p">(</span><span class="n">ucfg</span><span class="p">)</span>
|
|
<span class="k">for</span> <span class="n">upload</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">all_uploads</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="n">attrgetter</span><span class="p">(</span><span class="s2">"creation_time"</span><span class="p">)):</span>
|
|
<span class="n">ready</span> <span class="o">=</span> <span class="n">upload</span><span class="o">.</span><span class="n">status</span> <span class="o">==</span> <span class="s2">"READY"</span>
|
|
<span class="k">if</span> <span class="n">ready</span> <span class="ow">and</span> <span class="n">upload</span><span class="o">.</span><span class="n">uuid</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">pool_uuids</span><span class="p">:</span>
|
|
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Starting upload </span><span class="si">%s</span><span class="s2">..."</span><span class="p">,</span> <span class="n">upload</span><span class="o">.</span><span class="n">uuid</span><span class="p">)</span>
|
|
<span class="n">pool_uuids</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">upload</span><span class="o">.</span><span class="n">uuid</span><span class="p">)</span>
|
|
<span class="n">callback</span> <span class="o">=</span> <span class="n">remover</span><span class="p">(</span><span class="n">upload</span><span class="o">.</span><span class="n">uuid</span><span class="p">)</span>
|
|
<span class="n">pool</span><span class="o">.</span><span class="n">apply_async</span><span class="p">(</span>
|
|
<span class="n">upload</span><span class="o">.</span><span class="n">execute</span><span class="p">,</span>
|
|
<span class="p">(</span><span class="n">_write_callback</span><span class="p">(</span><span class="n">ucfg</span><span class="p">),),</span>
|
|
<span class="n">callback</span><span class="o">=</span><span class="n">callback</span><span class="p">,</span>
|
|
<span class="n">error_callback</span><span class="o">=</span><span class="n">callback</span><span class="p">,</span>
|
|
<span class="p">)</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
|
</pre></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
<footer>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<p>
|
|
© Copyright 2018, Red Hat, Inc.
|
|
|
|
</p>
|
|
</div>
|
|
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<script type="text/javascript">
|
|
jQuery(function () {
|
|
SphinxRtdTheme.Navigation.enable(true);
|
|
});
|
|
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</body>
|
|
</html> |