lorax/docs/html/_modules/pylorax/monitor.html
2018-06-04 16:50:39 -07:00

413 lines
28 KiB
HTML

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>pylorax.monitor &mdash; Lorax 28.13 documentation</title>
<link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<script src="../../_static/js/modernizr.min.js"></script>
</head>
<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
<div class="wy-side-scroll">
<div class="wy-side-nav-search">
<a href="../../index.html" class="icon icon-home"> Lorax
</a>
<div class="version">
28.13
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
<input type="text" name="q" placeholder="Search docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../intro.html">Introduction to Lorax</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../intro.html#before-lorax">Before Lorax</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../lorax.html">Lorax</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../livemedia-creator.html">livemedia-creator</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../lorax-composer.html">lorax-composer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../composer-cli.html">composer-cli</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../product-images.html">Product and Updates Images</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../modules.html">pylorax</a></li>
</ul>
</div>
</div>
</nav>
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
<nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../index.html">Lorax</a>
</nav>
<div class="wy-nav-content">
<div class="rst-content">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="wy-breadcrumbs">
<li><a href="../../index.html">Docs</a> &raquo;</li>
<li><a href="../index.html">Module code</a> &raquo;</li>
<li><a href="../pylorax.html">pylorax</a> &raquo;</li>
<li>pylorax.monitor</li>
<li class="wy-breadcrumbs-aside">
</li>
</ul>
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
<div itemprop="articleBody">
<h1>Source code for pylorax.monitor</h1><div class="highlight"><pre>
<span></span><span class="c1"># monitor.py</span>
<span class="c1">#</span>
<span class="c1"># Copyright (C) 2011-2015 Red Hat, Inc.</span>
<span class="c1">#</span>
<span class="c1"># This program is free software; you can redistribute it and/or modify</span>
<span class="c1"># it under the terms of the GNU General Public License as published by</span>
<span class="c1"># the Free Software Foundation; either version 2 of the License, or</span>
<span class="c1"># (at your option) any later version.</span>
<span class="c1">#</span>
<span class="c1"># This program is distributed in the hope that it will be useful,</span>
<span class="c1"># but WITHOUT ANY WARRANTY; without even the implied warranty of</span>
<span class="c1"># MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the</span>
<span class="c1"># GNU General Public License for more details.</span>
<span class="c1">#</span>
<span class="c1"># You should have received a copy of the GNU General Public License</span>
<span class="c1"># along with this program. If not, see &lt;http://www.gnu.org/licenses/&gt;.</span>
<span class="c1">#</span>
<span class="c1"># Author(s): Brian C. Lane &lt;bcl@redhat.com&gt;</span>
<span class="c1">#</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;livemedia-creator&quot;</span><span class="p">)</span>
<span class="kn">import</span> <span class="nn">re</span>
<span class="kn">import</span> <span class="nn">socket</span>
<span class="kn">import</span> <span class="nn">socketserver</span>
<span class="kn">import</span> <span class="nn">threading</span>
<span class="kn">import</span> <span class="nn">time</span>
<div class="viewcode-block" id="LogRequestHandler"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogRequestHandler">[docs]</a><span class="k">class</span> <span class="nc">LogRequestHandler</span><span class="p">(</span><span class="n">socketserver</span><span class="o">.</span><span class="n">BaseRequestHandler</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Handle monitoring and saving the logfiles from the virtual install</span>
<span class="sd"> Incoming data is written to self.server.log_path and each line is checked</span>
<span class="sd"> for patterns that would indicate that the installation failed.</span>
<span class="sd"> self.server.log_error is set True when this happens.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<div class="viewcode-block" id="LogRequestHandler.setup"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogRequestHandler.setup">[docs]</a> <span class="k">def</span> <span class="nf">setup</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Start writing to self.server.log_path&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">log_path</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">log_path</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="c1"># pylint: disable=attribute-defined-outside-init</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">settimeout</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span></div>
<div class="viewcode-block" id="LogRequestHandler.handle"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogRequestHandler.handle">[docs]</a> <span class="k">def</span> <span class="nf">handle</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Write incoming data to a logfile and check for errors</span>
<span class="sd"> Split incoming data into lines and check for any Tracebacks or other</span>
<span class="sd"> errors that indicate that the install failed.</span>
<span class="sd"> Loops until self.server.kill is True</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Processing logs from </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">client_address</span><span class="p">)</span>
<span class="n">line</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">kill</span><span class="p">:</span>
<span class="k">break</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">recv</span><span class="p">(</span><span class="mi">4096</span><span class="p">),</span> <span class="s2">&quot;utf8&quot;</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fp</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="c1"># check the data for errors and set error flag</span>
<span class="c1"># need to assemble it into lines so we can test for the error</span>
<span class="c1"># string.</span>
<span class="k">while</span> <span class="n">data</span><span class="p">:</span>
<span class="n">more</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="n">line</span> <span class="o">+=</span> <span class="n">more</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">more</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">iserror</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
<span class="n">line</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">more</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">data</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">except</span> <span class="n">socket</span><span class="o">.</span><span class="n">timeout</span><span class="p">:</span>
<span class="k">pass</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;log processing killed by exception: </span><span class="si">%s</span><span class="s2">&quot;</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
<span class="k">break</span></div>
<div class="viewcode-block" id="LogRequestHandler.finish"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogRequestHandler.finish">[docs]</a> <span class="k">def</span> <span class="nf">finish</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Shutting down log processing&quot;</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">fp</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">fp</span><span class="o">.</span><span class="n">close</span><span class="p">()</span></div>
<div class="viewcode-block" id="LogRequestHandler.iserror"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogRequestHandler.iserror">[docs]</a> <span class="k">def</span> <span class="nf">iserror</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">line</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Check a line to see if it contains an error indicating installation failure</span>
<span class="sd"> :param str line: log line to check for failure</span>
<span class="sd"> If the line contains IGNORED it will be skipped.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="s2">&quot;IGNORED&quot;</span> <span class="ow">in</span> <span class="n">line</span><span class="p">:</span>
<span class="k">return</span>
<span class="n">simple_tests</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;Traceback (&quot;</span><span class="p">,</span>
<span class="s2">&quot;Out of memory:&quot;</span><span class="p">,</span>
<span class="s2">&quot;Call Trace:&quot;</span><span class="p">,</span>
<span class="s2">&quot;insufficient disk space:&quot;</span><span class="p">,</span>
<span class="s2">&quot;Not enough disk space to download the packages&quot;</span><span class="p">,</span>
<span class="s2">&quot;error populating transaction after&quot;</span><span class="p">,</span>
<span class="s2">&quot;traceback script(s) have been run&quot;</span><span class="p">,</span>
<span class="s2">&quot;crashed on signal&quot;</span><span class="p">,</span>
<span class="s2">&quot;packaging: Missed: NoSuchPackage&quot;</span><span class="p">,</span>
<span class="s2">&quot;packaging: Installation failed&quot;</span><span class="p">,</span>
<span class="s2">&quot;The following error occurred while installing. This is a fatal error&quot;</span><span class="p">]</span>
<span class="n">re_tests</span> <span class="o">=</span> <span class="p">[</span><span class="sa">r</span><span class="s2">&quot;packaging: base repo .* not valid&quot;</span><span class="p">,</span>
<span class="sa">r</span><span class="s2">&quot;packaging: .* requires .*&quot;</span><span class="p">]</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">simple_tests</span><span class="p">:</span>
<span class="k">if</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">line</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">log_error</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">error_line</span> <span class="o">=</span> <span class="n">line</span>
<span class="k">return</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">re_tests</span><span class="p">:</span>
<span class="k">if</span> <span class="n">re</span><span class="o">.</span><span class="n">search</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">line</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">log_error</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">error_line</span> <span class="o">=</span> <span class="n">line</span>
<span class="k">return</span></div></div>
<div class="viewcode-block" id="LogServer"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogServer">[docs]</a><span class="k">class</span> <span class="nc">LogServer</span><span class="p">(</span><span class="n">socketserver</span><span class="o">.</span><span class="n">TCPServer</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;A TCP Server that listens for log data&quot;&quot;&quot;</span>
<span class="c1"># Number of seconds to wait for a connection after startup</span>
<span class="n">timeout</span> <span class="o">=</span> <span class="mi">60</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">log_path</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Setup the log server</span>
<span class="sd"> :param str log_path: Path to the log file to write</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">kill</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log_error</span> <span class="o">=</span> <span class="kc">False</span>
<span class="bp">self</span><span class="o">.</span><span class="n">error_line</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log_path</span> <span class="o">=</span> <span class="n">log_path</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">&quot;timeout&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
<span class="n">socketserver</span><span class="o">.</span><span class="n">TCPServer</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<div class="viewcode-block" id="LogServer.log_check"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogServer.log_check">[docs]</a> <span class="k">def</span> <span class="nf">log_check</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Check to see if an error has been found in the log</span>
<span class="sd"> :returns: True if there has been an error</span>
<span class="sd"> :rtype: bool</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span><span class="p">:</span>
<span class="n">taking_too_long</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_start_time</span> <span class="o">+</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_timeout</span> <span class="o">*</span> <span class="mi">60</span><span class="p">)</span>
<span class="k">if</span> <span class="n">taking_too_long</span><span class="p">:</span>
<span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">&quot;Canceling installation due to timeout&quot;</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">taking_too_long</span> <span class="o">=</span> <span class="kc">False</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">log_error</span> <span class="ow">or</span> <span class="n">taking_too_long</span></div></div>
<div class="viewcode-block" id="LogMonitor"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogMonitor">[docs]</a><span class="k">class</span> <span class="nc">LogMonitor</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Setup a server to monitor the logs output by the installation</span>
<span class="sd"> This needs to be running before the virt-install runs, it expects</span>
<span class="sd"> there to be a listener on the port used for the virtio log port.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">log_path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">host</span><span class="o">=</span><span class="s2">&quot;localhost&quot;</span><span class="p">,</span> <span class="n">port</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Start a thread to monitor the logs.</span>
<span class="sd"> :param str log_path: Path to the logfile to write</span>
<span class="sd"> :param str host: Host to bind to. Default is localhost.</span>
<span class="sd"> :param int port: Port to listen to or 0 to pick a port</span>
<span class="sd"> If 0 is passed for the port the dynamically assigned port will be</span>
<span class="sd"> available as self.port</span>
<span class="sd"> If log_path isn&#39;t set then it only monitors the logs, instead of</span>
<span class="sd"> also writing them to disk.</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span> <span class="o">=</span> <span class="n">LogServer</span><span class="p">(</span><span class="n">log_path</span><span class="p">,</span> <span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">),</span> <span class="n">LogRequestHandler</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">port</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">server_address</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log_path</span> <span class="o">=</span> <span class="n">log_path</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server_thread</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">handle_request</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server_thread</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server_thread</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<div class="viewcode-block" id="LogMonitor.shutdown"><a class="viewcode-back" href="../../pylorax.html#pylorax.monitor.LogMonitor.shutdown">[docs]</a> <span class="k">def</span> <span class="nf">shutdown</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;Force shutdown of the monitoring thread&quot;&quot;&quot;</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">kill</span> <span class="o">=</span> <span class="kc">True</span>
<span class="bp">self</span><span class="o">.</span><span class="n">server_thread</span><span class="o">.</span><span class="n">join</span><span class="p">()</span></div></div>
</pre></div>
</div>
</div>
<footer>
<hr/>
<div role="contentinfo">
<p>
&copy; Copyright 2018, Red Hat, Inc..
</p>
</div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
</div>
</div>
</section>
</div>
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../',
VERSION:'28.13',
LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
<script type="text/javascript" src="../../_static/underscore.js"></script>
<script type="text/javascript" src="../../_static/doctools.js"></script>
<script type="text/javascript" src="../../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
</body>
</html>