From 893791cf7d146c733214e7d89a877aac274afaf5 Mon Sep 17 00:00:00 2001 From: Chris Lumens Date: Wed, 27 May 2015 13:17:38 -0400 Subject: [PATCH] Break all the log monitoring stuff from LMC out into its own module. --- src/pylorax/monitor.py | 191 +++++++++++++++++++++++++++++++++++++ src/sbin/livemedia-creator | 171 +-------------------------------- 2 files changed, 192 insertions(+), 170 deletions(-) create mode 100644 src/pylorax/monitor.py diff --git a/src/pylorax/monitor.py b/src/pylorax/monitor.py new file mode 100644 index 00000000..f6acaee4 --- /dev/null +++ b/src/pylorax/monitor.py @@ -0,0 +1,191 @@ +# monitor.py +# +# Copyright (C) 2011-2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author(s): Brian C. Lane +# +import logging +log = logging.getLogger("livemedia-creator") + +import re +import socket +import socketserver +import threading +import time + +class LogRequestHandler(socketserver.BaseRequestHandler): + """ + Handle monitoring and saving the logfiles from the virtual install + + Incoming data is written to self.server.log_path and each line is checked + for patterns that would indicate that the installation failed. + self.server.log_error is set True when this happens. + """ + def setup(self): + """Start writing to self.server.log_path""" + + if self.server.log_path: + self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init + else: + self.fp = None + self.request.settimeout(10) + + def handle(self): + """ + Write incoming data to a logfile and check for errors + + Split incoming data into lines and check for any Tracebacks or other + errors that indicate that the install failed. + + Loops until self.server.kill is True + """ + log.info("Processing logs from %s", self.client_address) + line = "" + while True: + if self.server.kill: + break + + try: + data = str(self.request.recv(4096), "utf8") + if self.fp: + self.fp.write(data) + self.fp.flush() + + # check the data for errors and set error flag + # need to assemble it into lines so we can test for the error + # string. + while data: + more = data.split("\n", 1) + line += more[0] + if len(more) > 1: + self.iserror(line) + line = "" + data = more[1] + else: + data = None + + except socket.timeout: + pass + except Exception as e: # pylint: disable=broad-except + log.info("log processing killed by exception: %s", e) + break + + def finish(self): + log.info("Shutting down log processing") + self.request.close() + if self.fp: + self.fp.close() + + def iserror(self, line): + """ + Check a line to see if it contains an error indicating installation failure + + :param str line: log line to check for failure + + If the line contains IGNORED it will be skipped. + """ + if "IGNORED" in line: + return + simple_tests = ["Traceback (", + "Out of memory:", + "Call Trace:", + "insufficient disk space:", + "error populating transaction after", + "traceback script(s) have been run", + "packaging: Missed: NoSuchPackage"] + re_tests = [r"packaging: base repo .* not valid", + r"packaging: .* requires .*"] + for t in simple_tests: + if t in line: + self.server.log_error = True + self.server.error_line = line + return + for t in re_tests: + if re.search(t, line): + self.server.log_error = True + self.server.error_line = line + return + + +class LogServer(socketserver.TCPServer): + """A TCP Server that listens for log data""" + + # Number of seconds to wait for a connection after startup + timeout = 60 + + def __init__(self, log_path, *args, **kwargs): + """ + Setup the log server + + :param str log_path: Path to the log file to write + """ + self.kill = False + self.log_error = False + self.error_line = "" + self.log_path = log_path + self._timeout = kwargs.pop("timeout", None) + if self._timeout: + self._start_time = time.time() + socketserver.TCPServer.__init__(self, *args, **kwargs) + + def log_check(self): + """ + Check to see if an error has been found in the log + + :returns: True if there has been an error + :rtype: bool + """ + if self._timeout: + taking_too_long = time.time() > self._start_time + (self._timeout * 60) + if taking_too_long: + log.error("Canceling installation due to timeout") + else: + taking_too_long = False + return self.log_error or taking_too_long + + +class LogMonitor(object): + """ + Setup a server to monitor the logs output by the installation + + This needs to be running before the virt-install runs, it expects + there to be a listener on the port used for the virtio log port. + """ + def __init__(self, log_path=None, host="localhost", port=0, timeout=None): + """ + Start a thread to monitor the logs. + + :param str log_path: Path to the logfile to write + :param str host: Host to bind to. Default is localhost. + :param int port: Port to listen to or 0 to pick a port + + If 0 is passed for the port the dynamically assigned port will be + available as self.port + + If log_path isn't set then it only monitors the logs, instead of + also writing them to disk. + """ + self.server = LogServer(log_path, (host, port), LogRequestHandler, timeout=timeout) + self.host, self.port = self.server.server_address + self.log_path = log_path + self.server_thread = threading.Thread(target=self.server.handle_request) + self.server_thread.daemon = True + self.server_thread.start() + + def shutdown(self): + """Force shutdown of the monitoring thread""" + self.server.kill = True + self.server_thread.join() diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 108871bb..482d53bd 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -29,15 +29,10 @@ import sys import uuid import tempfile import subprocess -import socket -import threading -import socketserver -import time from time import sleep import shutil import argparse import hashlib -import re import glob # Use pykickstart to calculate disk image size @@ -59,6 +54,7 @@ from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount from pylorax.imgutils import mksquashfs, mkqcow2, mktar, mkrootfsimg from pylorax.imgutils import copytree from pylorax.executils import execWithRedirect, execWithCapture, runcmd +from pylorax.monitor import LogMonitor # no-virt mode doesn't need libvirt, so make it optional try: @@ -77,171 +73,6 @@ class InstallError(Exception): pass -class LogRequestHandler(socketserver.BaseRequestHandler): - """ - Handle monitoring and saving the logfiles from the virtual install - - Incoming data is written to self.server.log_path and each line is checked - for patterns that would indicate that the installation failed. - self.server.log_error is set True when this happens. - """ - def setup(self): - """Start writing to self.server.log_path""" - - if self.server.log_path: - self.fp = open(self.server.log_path, "w") # pylint: disable=attribute-defined-outside-init - else: - self.fp = None - self.request.settimeout(10) - - def handle(self): - """ - Write incoming data to a logfile and check for errors - - Split incoming data into lines and check for any Tracebacks or other - errors that indicate that the install failed. - - Loops until self.server.kill is True - """ - log.info("Processing logs from %s", self.client_address) - line = "" - while True: - if self.server.kill: - break - - try: - data = str(self.request.recv(4096), "utf8") - if self.fp: - self.fp.write(data) - self.fp.flush() - - # check the data for errors and set error flag - # need to assemble it into lines so we can test for the error - # string. - while data: - more = data.split("\n", 1) - line += more[0] - if len(more) > 1: - self.iserror(line) - line = "" - data = more[1] - else: - data = None - - except socket.timeout: - pass - except Exception as e: # pylint: disable=broad-except - log.info("log processing killed by exception: %s", e) - break - - def finish(self): - log.info("Shutting down log processing") - self.request.close() - if self.fp: - self.fp.close() - - def iserror(self, line): - """ - Check a line to see if it contains an error indicating installation failure - - :param str line: log line to check for failure - - If the line contains IGNORED it will be skipped. - """ - if "IGNORED" in line: - return - simple_tests = ["Traceback (", - "Out of memory:", - "Call Trace:", - "insufficient disk space:", - "error populating transaction after", - "traceback script(s) have been run", - "packaging: Missed: NoSuchPackage"] - re_tests = [r"packaging: base repo .* not valid", - r"packaging: .* requires .*"] - for t in simple_tests: - if t in line: - self.server.log_error = True - self.server.error_line = line - return - for t in re_tests: - if re.search(t, line): - self.server.log_error = True - self.server.error_line = line - return - - -class LogServer(socketserver.TCPServer): - """A TCP Server that listens for log data""" - - # Number of seconds to wait for a connection after startup - timeout = 60 - - def __init__(self, log_path, *args, **kwargs): - """ - Setup the log server - - :param str log_path: Path to the log file to write - """ - self.kill = False - self.log_error = False - self.error_line = "" - self.log_path = log_path - self._timeout = kwargs.pop("timeout", None) - if self._timeout: - self._start_time = time.time() - socketserver.TCPServer.__init__(self, *args, **kwargs) - - def log_check(self): - """ - Check to see if an error has been found in the log - - :returns: True if there has been an error - :rtype: bool - """ - if self._timeout: - taking_too_long = time.time() > self._start_time + (self._timeout * 60) - if taking_too_long: - log.error("Canceling installation due to timeout") - else: - taking_too_long = False - return self.log_error or taking_too_long - - -class LogMonitor(object): - """ - Setup a server to monitor the logs output by the installation - - This needs to be running before the virt-install runs, it expects - there to be a listener on the port used for the virtio log port. - """ - def __init__(self, log_path=None, host="localhost", port=0, timeout=None): - """ - Start a thread to monitor the logs. - - :param str log_path: Path to the logfile to write - :param str host: Host to bind to. Default is localhost. - :param int port: Port to listen to or 0 to pick a port - - If 0 is passed for the port the dynamically assigned port will be - available as self.port - - If log_path isn't set then it only monitors the logs, instead of - also writing them to disk. - """ - self.server = LogServer(log_path, (host, port), LogRequestHandler, timeout=timeout) - self.host, self.port = self.server.server_address - self.log_path = log_path - self.server_thread = threading.Thread(target=self.server.handle_request) - self.server_thread.daemon = True - self.server_thread.start() - - def shutdown(self): - """Force shutdown of the monitoring thread""" - self.server.kill = True - self.server_thread.join() - - class IsoMountpoint(object): """ Mount the iso and check to make sure the vmlinuz and initrd.img files exist