diff --git a/lorax-composer/.buildinfo b/lorax-composer/.buildinfo index 82148231..1f43aec2 100644 --- a/lorax-composer/.buildinfo +++ b/lorax-composer/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 50e837e886c9cb44ee8f8828f75d7ded +config: 12d527250403177f77175f9f02bc8463 tags: fbb0d17656682115ca4d033fb2f83ba1 diff --git a/lorax-composer/.doctrees/composer-cli.doctree b/lorax-composer/.doctrees/composer-cli.doctree index 3905741a..70f36a3b 100644 Binary files a/lorax-composer/.doctrees/composer-cli.doctree and b/lorax-composer/.doctrees/composer-cli.doctree differ diff --git a/lorax-composer/.doctrees/composer.cli.doctree b/lorax-composer/.doctrees/composer.cli.doctree index 01939c6c..bc2cdfd5 100644 Binary files a/lorax-composer/.doctrees/composer.cli.doctree and b/lorax-composer/.doctrees/composer.cli.doctree differ diff --git a/lorax-composer/.doctrees/composer.doctree b/lorax-composer/.doctrees/composer.doctree index 945711d0..fa0f9202 100644 Binary files a/lorax-composer/.doctrees/composer.doctree and b/lorax-composer/.doctrees/composer.doctree differ diff --git a/lorax-composer/.doctrees/environment.pickle b/lorax-composer/.doctrees/environment.pickle index a687a431..5bd8ab3f 100644 Binary files a/lorax-composer/.doctrees/environment.pickle and b/lorax-composer/.doctrees/environment.pickle differ diff --git a/lorax-composer/.doctrees/index.doctree b/lorax-composer/.doctrees/index.doctree index c2101334..e32d0913 100644 Binary files a/lorax-composer/.doctrees/index.doctree and b/lorax-composer/.doctrees/index.doctree differ diff --git a/lorax-composer/.doctrees/intro.doctree b/lorax-composer/.doctrees/intro.doctree index c35bf25f..e606f96c 100644 Binary files a/lorax-composer/.doctrees/intro.doctree and b/lorax-composer/.doctrees/intro.doctree differ diff --git a/lorax-composer/.doctrees/livemedia-creator.doctree b/lorax-composer/.doctrees/livemedia-creator.doctree index 7d19a7cb..81771a97 100644 Binary files a/lorax-composer/.doctrees/livemedia-creator.doctree and b/lorax-composer/.doctrees/livemedia-creator.doctree differ diff --git a/lorax-composer/.doctrees/lorax-composer.doctree b/lorax-composer/.doctrees/lorax-composer.doctree index 6c3c5445..fd83fc08 100644 Binary files a/lorax-composer/.doctrees/lorax-composer.doctree and b/lorax-composer/.doctrees/lorax-composer.doctree differ diff --git a/lorax-composer/.doctrees/lorax.doctree b/lorax-composer/.doctrees/lorax.doctree index b7e2fed3..22f2a8b7 100644 Binary files a/lorax-composer/.doctrees/lorax.doctree and b/lorax-composer/.doctrees/lorax.doctree differ diff --git a/lorax-composer/.doctrees/modules.doctree b/lorax-composer/.doctrees/modules.doctree index 6dc44e6e..1e631d6e 100644 Binary files a/lorax-composer/.doctrees/modules.doctree and b/lorax-composer/.doctrees/modules.doctree differ diff --git a/lorax-composer/.doctrees/product-images.doctree b/lorax-composer/.doctrees/product-images.doctree index fd60188b..4353d99e 100644 Binary files a/lorax-composer/.doctrees/product-images.doctree and b/lorax-composer/.doctrees/product-images.doctree differ diff --git a/lorax-composer/.doctrees/pylorax.api.doctree b/lorax-composer/.doctrees/pylorax.api.doctree index 0c62ad4d..7b63d85b 100644 Binary files a/lorax-composer/.doctrees/pylorax.api.doctree and b/lorax-composer/.doctrees/pylorax.api.doctree differ diff --git a/lorax-composer/.doctrees/pylorax.doctree b/lorax-composer/.doctrees/pylorax.doctree index 9f25122d..37e0ec42 100644 Binary files a/lorax-composer/.doctrees/pylorax.doctree and b/lorax-composer/.doctrees/pylorax.doctree differ diff --git a/lorax-composer/_modules/composer/cli.html b/lorax-composer/_modules/composer/cli.html index 4effe024..1e5d84ad 100644 --- a/lorax-composer/_modules/composer/cli.html +++ b/lorax-composer/_modules/composer/cli.html @@ -8,7 +8,7 @@ - composer.cli — Lorax 19.7.22 documentation + composer.cli — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • @@ -138,7 +138,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • diff --git a/lorax-composer/_modules/composer/cli/blueprints.html b/lorax-composer/_modules/composer/cli/blueprints.html index ca81ac8c..11bdaa0a 100644 --- a/lorax-composer/_modules/composer/cli/blueprints.html +++ b/lorax-composer/_modules/composer/cli/blueprints.html @@ -8,7 +8,7 @@ - composer.cli.blueprints — Lorax 19.7.22 documentation + composer.cli.blueprints — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -239,62 +239,129 @@ return rc for diff in result["diff"]: - print(prettyDiffEntry(diff)) + print(pretty_diff_entry(diff)) return rc -
    [docs]def prettyDiffEntry(diff): +
    [docs]def pretty_dict(d): + """Return the dict as a human readable single line + + :param d: key/values + :type d: dict + :returns: String of the dict's keys and values + :rtype: str + + key="str", key="str1,str2", ... + """ + result = [] + for k in sorted(d.keys()): + if type(d[k]) == type(""): + result.append('%s="%s"' % (k, d[k])) + elif type(d[k]) == type([]) and type(d[k][0]) == type(""): + result.append('%s="%s"' % (k, ", ".join(d[k]))) + elif type(d[k]) == type([]) and type(d[k][0]) == type({}): + result.append('%s="%s"' % (k, pretty_dict(d[k]))) + return " ".join(result) +
    +
    [docs]def dict_names(lst): + """Return comma-separated list of the dict's name/user fields + + :param d: key/values + :type d: dict + :returns: String of the dict's keys and values + :rtype: str + + root, norm + """ + if "user" in lst[0]: + field_name = "user" + elif "name" in lst[0]: + field_name = "name" + else: + # Use first fields in sorted keys + field_name = sorted(lst[0].keys())[0] + + return ", ".join(d[field_name] for d in lst) +
    +
    [docs]def pretty_diff_entry(diff): """Generate nice diff entry string. :param diff: Difference entry dict :type diff: dict :returns: Nice string """ - def change(diff): - if diff["old"] and diff["new"]: - return "Changed" - elif diff["new"] and not diff["old"]: - return "Added" - elif diff["old"] and not diff["new"]: - return "Removed" - else: - return "Unknown" + if diff["old"] and diff["new"]: + change = "Changed" + elif diff["new"] and not diff["old"]: + change = "Added" + elif diff["old"] and not diff["new"]: + change = "Removed" + else: + change = "Unknown" - def name(diff): - if diff["old"]: - return diff["old"].keys()[0] - elif diff["new"]: - return diff["new"].keys()[0] - else: - return "Unknown" + if diff["old"]: + name = list(diff["old"].keys())[0] + elif diff["new"]: + name = list(diff["new"].keys())[0] + else: + name = "Unknown" def details(diff): - if change(diff) == "Changed": - if name(diff) == "Description": - return '"%s" -> "%s"' % (diff["old"][name(diff)], diff["new"][name(diff)]) - elif name(diff) == "Version": - return "%s -> %s" % (diff["old"][name(diff)], diff["new"][name(diff)]) - elif name(diff) in ["Module", "Package"]: - return "%s %s -> %s" % (diff["old"][name(diff)]["name"], diff["old"][name(diff)]["version"], - diff["new"][name(diff)]["version"]) + if change == "Changed": + if type(diff["old"][name]) == type(""): + if name == "Description" or " " in diff["old"][name]: + return '"%s" -> "%s"' % (diff["old"][name], diff["new"][name]) + else: + return "%s -> %s" % (diff["old"][name], diff["new"][name]) + elif name in ["Module", "Package"]: + return "%s %s -> %s" % (diff["old"][name]["name"], diff["old"][name]["version"], + diff["new"][name]["version"]) + elif type(diff["old"][name]) == type([]): + if type(diff["old"][name][0]) == type(""): + return "%s -> %s" % (" ".join(diff["old"][name]), " ".join(diff["new"][name])) + elif type(diff["old"][name][0]) == type({}): + # Lists of dicts are too long to display in detail, just show their names + return "%s -> %s" % (dict_names(diff["old"][name]), dict_names(diff["new"][name])) + elif type(diff["old"][name]) == type({}): + return "%s -> %s" % (pretty_dict(diff["old"][name]), pretty_dict(diff["new"][name])) else: return "Unknown" - elif change(diff) == "Added": - if name(diff) in ["Module", "Package"]: - return "%s %s" % (diff["new"][name(diff)]["name"], diff["new"][name(diff)]["version"]) - elif name(diff) in ["Group"]: - return diff["new"][name(diff)]["name"] + elif change == "Added": + if name in ["Module", "Package"]: + return "%s %s" % (diff["new"][name]["name"], diff["new"][name]["version"]) + elif name in ["Group"]: + return diff["new"][name]["name"] + elif type(diff["new"][name]) == type(""): + return diff["new"][name] + elif type(diff["new"][name]) == type([]): + if type(diff["new"][name][0]) == type(""): + return " ".join(diff["new"][name]) + elif type(diff["new"][name][0]) == type({}): + # Lists of dicts are too long to display in detail, just show their names + return dict_names(diff["new"][name]) + elif type(diff["new"][name]) == type({}): + return pretty_dict(diff["new"][name]) else: - return " ".join([diff["new"][k] for k in diff["new"]]) - elif change(diff) == "Removed": - if name(diff) in ["Module", "Package"]: - return "%s %s" % (diff["old"][name(diff)]["name"], diff["old"][name(diff)]["version"]) - elif name(diff) in ["Group"]: - return diff["old"][name(diff)]["name"] + return "unknown/todo: %s" % type(diff["new"][name]) + elif change == "Removed": + if name in ["Module", "Package"]: + return "%s %s" % (diff["old"][name]["name"], diff["old"][name]["version"]) + elif name in ["Group"]: + return diff["old"][name]["name"] + elif type(diff["old"][name]) == type(""): + return diff["old"][name] + elif type(diff["old"][name]) == type([]): + if type(diff["old"][name][0]) == type(""): + return " ".join(diff["old"][name]) + elif type(diff["old"][name][0]) == type({}): + # Lists of dicts are too long to display in detail, just show their names + return dict_names(diff["old"][name]) + elif type(diff["old"][name]) == type({}): + return pretty_dict(diff["old"][name]) else: - return " ".join([diff["old"][k] for k in diff["old"]]) + return "unknown/todo: %s" % type(diff["new"][name]) - return change(diff) + " " + name(diff) + " " + details(diff) + return change + " " + name + " " + details(diff)
    [docs]def blueprints_save(socket_path, api_version, args, show_json=False): """Save the blueprint to a TOML file @@ -594,7 +661,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/compose.html b/lorax-composer/_modules/composer/cli/compose.html index 89cb1cfa..9cffe38a 100644 --- a/lorax-composer/_modules/composer/cli/compose.html +++ b/lorax-composer/_modules/composer/cli/compose.html @@ -8,7 +8,7 @@ - composer.cli.compose — Lorax 19.7.22 documentation + composer.cli.compose — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -596,7 +596,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/modules.html b/lorax-composer/_modules/composer/cli/modules.html index 3d3f144a..8605761a 100644 --- a/lorax-composer/_modules/composer/cli/modules.html +++ b/lorax-composer/_modules/composer/cli/modules.html @@ -8,7 +8,7 @@ - composer.cli.modules — Lorax 19.7.22 documentation + composer.cli.modules — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -131,7 +131,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/projects.html b/lorax-composer/_modules/composer/cli/projects.html index d971f56e..6da14436 100644 --- a/lorax-composer/_modules/composer/cli/projects.html +++ b/lorax-composer/_modules/composer/cli/projects.html @@ -8,7 +8,7 @@ - composer.cli.projects — Lorax 19.7.22 documentation + composer.cli.projects — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -193,7 +193,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/sources.html b/lorax-composer/_modules/composer/cli/sources.html index e033dcef..adf1cebf 100644 --- a/lorax-composer/_modules/composer/cli/sources.html +++ b/lorax-composer/_modules/composer/cli/sources.html @@ -8,7 +8,7 @@ - composer.cli.sources — Lorax 19.7.22 documentation + composer.cli.sources — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -235,7 +235,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/status.html b/lorax-composer/_modules/composer/cli/status.html index fcf31e32..c235ddcb 100644 --- a/lorax-composer/_modules/composer/cli/status.html +++ b/lorax-composer/_modules/composer/cli/status.html @@ -8,7 +8,7 @@ - composer.cli.status — Lorax 19.7.22 documentation + composer.cli.status — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -139,7 +139,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/cli/utilities.html b/lorax-composer/_modules/composer/cli/utilities.html index b703b564..e7355acf 100644 --- a/lorax-composer/_modules/composer/cli/utilities.html +++ b/lorax-composer/_modules/composer/cli/utilities.html @@ -8,7 +8,7 @@ - composer.cli.utilities — Lorax 19.7.22 documentation + composer.cli.utilities — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • @@ -178,7 +178,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • composer.cli »
  • diff --git a/lorax-composer/_modules/composer/http_client.html b/lorax-composer/_modules/composer/http_client.html index 4cb4a5ec..30c6e540 100644 --- a/lorax-composer/_modules/composer/http_client.html +++ b/lorax-composer/_modules/composer/http_client.html @@ -8,7 +8,7 @@ - composer.http_client — Lorax 19.7.22 documentation + composer.http_client — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • @@ -342,7 +342,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • diff --git a/lorax-composer/_modules/composer/unix_socket.html b/lorax-composer/_modules/composer/unix_socket.html index 6d20630d..01a47064 100644 --- a/lorax-composer/_modules/composer/unix_socket.html +++ b/lorax-composer/_modules/composer/unix_socket.html @@ -8,7 +8,7 @@ - composer.unix_socket — Lorax 19.7.22 documentation + composer.unix_socket — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • @@ -77,7 +77,7 @@ # https://github.com/docker/docker-py/blob/master/docker/transport/unixconn.py
    [docs]class UnixHTTPConnection(httplib.HTTPConnection, object): - def __init__(self, socket_path, timeout=60): + def __init__(self, socket_path, timeout=60*5): """Create an HTTP connection to a unix domain socket :param socket_path: The path to the Unix domain socket @@ -99,13 +99,15 @@
    [docs]class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool): - def __init__(self, socket_path, timeout=60): + def __init__(self, socket_path, timeout=60*5): """Create a connection pool using a Unix domain socket :param socket_path: The path to the Unix domain socket :param timeout: Number of seconds to timeout the connection + + NOTE: retries are disabled for these connections, they are never useful """ - super(UnixHTTPConnectionPool, self).__init__('localhost', timeout=timeout) + super(UnixHTTPConnectionPool, self).__init__('localhost', timeout=timeout, retries=False) self.socket_path = socket_path def _new_conn(self): @@ -143,7 +145,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • diff --git a/lorax-composer/_modules/index.html b/lorax-composer/_modules/index.html index 448b3c0d..061ba6da 100644 --- a/lorax-composer/_modules/index.html +++ b/lorax-composer/_modules/index.html @@ -8,7 +8,7 @@ - Overview: module code — Lorax 19.7.22 documentation + Overview: module code — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -59,7 +59,8 @@
  • composer.unix_socket
  • pylorax
  • diff --git a/lorax-composer/_modules/pylorax/api.html b/lorax-composer/_modules/pylorax/api.html index 7fa4df34..1b3b7b84 100644 --- a/lorax-composer/_modules/pylorax/api.html +++ b/lorax-composer/_modules/pylorax/api.html @@ -8,7 +8,7 @@ - pylorax.api — Lorax 19.7.22 documentation + pylorax.api — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -109,7 +109,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/api/bisect.html b/lorax-composer/_modules/pylorax/api/bisect.html new file mode 100644 index 00000000..adcb18a4 --- /dev/null +++ b/lorax-composer/_modules/pylorax/api/bisect.html @@ -0,0 +1,147 @@ + + + + + + + + + + pylorax.api.bisect — Lorax 19.7.43 documentation + + + + + + + + + + + + + + +
    +
    +
    +
    + +

    Source code for pylorax.api.bisect

    +#
    +# Copyright (C) 2018 Red Hat, Inc.
    +#
    +# This program is free software; you can redistribute it and/or modify
    +# it under the terms of the GNU General Public License as published by
    +# the Free Software Foundation; either version 2 of the License, or
    +# (at your option) any later version.
    +#
    +# This program is distributed in the hope that it will be useful,
    +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    +# GNU General Public License for more details.
    +#
    +# You should have received a copy of the GNU General Public License
    +# along with this program.  If not, see <http://www.gnu.org/licenses/>.
    +#
    +
    [docs]def insort_left(a, x, key=None, lo=0, hi=None): + """Insert item x in list a, and keep it sorted assuming a is sorted. + + :param a: sorted list + :type a: list + :param x: item to insert into the list + :type x: object + :param key: Function to use to compare items in the list + :type key: function + :returns: index where the item was inserted + :rtype: int + + If x is already in a, insert it to the left of the leftmost x. + Optional args lo (default 0) and hi (default len(a)) bound the + slice of a to be searched. + + This is a modified version of bisect.insort_left that can use a + function for the compare, and returns the index position where it + was inserted. + """ + if key is None: + key = lambda i: i + + if lo < 0: + raise ValueError('lo must be non-negative') + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if key(a[mid]) < key(x): lo = mid+1 + else: hi = mid + a.insert(lo, x) + return lo
    +
    + +
    +
    +
    +
    +
    + + +
    +
    +
    +
    + + + + \ No newline at end of file diff --git a/lorax-composer/_modules/pylorax/api/checkparams.html b/lorax-composer/_modules/pylorax/api/checkparams.html index 7024cfc4..068a1dd6 100644 --- a/lorax-composer/_modules/pylorax/api/checkparams.html +++ b/lorax-composer/_modules/pylorax/api/checkparams.html @@ -8,7 +8,7 @@ - pylorax.api.checkparams — Lorax 19.7.22 documentation + pylorax.api.checkparams — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -128,7 +128,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/compose.html b/lorax-composer/_modules/pylorax/api/compose.html index 76e65264..131bf24e 100644 --- a/lorax-composer/_modules/pylorax/api/compose.html +++ b/lorax-composer/_modules/pylorax/api/compose.html @@ -8,7 +8,7 @@ - pylorax.api.compose — Lorax 19.7.22 documentation + pylorax.api.compose — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -51,7 +51,7 @@

    Source code for pylorax.api.compose

    -# Copyright (C) 2018 Red Hat, Inc.
    +# Copyright (C) 2018-2019 Red Hat, Inc.
     #
     # This program is free software; you can redistribute it and/or modify
     # it under the terms of the GNU General Public License as published by
    @@ -83,11 +83,13 @@
     parameters needed to generate the desired output. Other types should be set to False.
     
     """
    +from __future__ import print_function
     import logging
     log = logging.getLogger("lorax-composer")
     
     import os
     from glob import glob
    +from StringIO import StringIO
     from math import ceil
     import pytoml as toml
     import shutil
    @@ -103,6 +105,7 @@
     from pylorax.api.projects import ProjectsError
     from pylorax.api.recipes import read_recipe_and_id
     from pylorax.api.timestamp import TS_CREATED, write_timestamp
    +from pylorax.executils import runcmd_output
     from pylorax.imgutils import default_image_name
     from pylorax.sysutils import joinpaths
     
    @@ -172,9 +175,391 @@
         if not r.sslverify:
             cmd += '--noverifyssl'
     
    +    if r.sslcacert:
    +        cmd += ' --sslcacert="%s"' % r.sslcacert
    +    if r.sslclientcert:
    +        cmd += ' --sslclientcert="%s"' % r.sslclientcert
    +    if r.sslclientkey:
    +        cmd += ' --sslclientkey="%s"' % r.sslclientkey
    +
         return cmd
     
     
    +
    [docs]def bootloader_append(line, kernel_append): + """ Insert the kernel_append string into the --append argument + + :param line: The bootloader ... line + :type line: str + :param kernel_append: The arguments to append to the --append section + :type kernel_append: str + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + if ks.handler.bootloader.appendLine: + ks.handler.bootloader.appendLine += " %s" % kernel_append + else: + ks.handler.bootloader.appendLine = kernel_append + + # Converting back to a string includes a comment, return just the bootloader line + return str(ks.handler.bootloader).splitlines()[-1] + +
    +
    [docs]def get_kernel_append(recipe): + """Return the customizations.kernel append value + + :param recipe: + :type recipe: Recipe object + :returns: append value or empty string + :rtype: str + """ + if "customizations" not in recipe or \ + "kernel" not in recipe["customizations"] or \ + "append" not in recipe["customizations"]["kernel"]: + return "" + return recipe["customizations"]["kernel"]["append"] + +
    +
    [docs]def timezone_cmd(line, settings): + """ Update the timezone line with the settings + + :param line: The timezone ... line + :type line: str + :param settings: A dict with timezone and/or ntpservers list + :type settings: dict + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + if "timezone" in settings: + ks.handler.timezone.timezone = settings["timezone"] + if "ntpservers" in settings: + ks.handler.timezone.ntpservers = settings["ntpservers"] + + # Converting back to a string includes a comment, return just the timezone line + return str(ks.handler.timezone).splitlines()[-1] + +
    +
    [docs]def get_timezone_settings(recipe): + """Return the customizations.timezone dict + + :param recipe: + :type recipe: Recipe object + :returns: append value or empty string + :rtype: dict + """ + if "customizations" not in recipe or \ + "timezone" not in recipe["customizations"]: + return {} + return recipe["customizations"]["timezone"] + +
    +
    [docs]def lang_cmd(line, languages): + """ Update the lang line with the languages + + :param line: The lang ... line + :type line: str + :param settings: The list of languages + :type settings: list + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + if languages: + ks.handler.lang.lang = languages[0] + + if len(languages) > 1: + ks.handler.lang.addsupport = languages[1:] + + # Converting back to a string includes a comment, return just the lang line + return str(ks.handler.lang).splitlines()[-1] + +
    +
    [docs]def get_languages(recipe): + """Return the customizations.locale.languages list + + :param recipe: The recipe + :type recipe: Recipe object + :returns: list of language strings + :rtype: list + """ + if "customizations" not in recipe or \ + "locale" not in recipe["customizations"] or \ + "languages" not in recipe["customizations"]["locale"]: + return [] + return recipe["customizations"]["locale"]["languages"] + +
    +
    [docs]def keyboard_cmd(line, layout): + """ Update the keyboard line with the layout + + :param line: The keyboard ... line + :type line: str + :param settings: The keyboard layout + :type settings: str + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + if layout: + ks.handler.keyboard.keyboard = layout + ks.handler.keyboard.vc_keymap = "" + ks.handler.keyboard.x_layouts = [] + + # Converting back to a string includes a comment, return just the keyboard line + return str(ks.handler.keyboard).splitlines()[-1] + +
    +
    [docs]def get_keyboard_layout(recipe): + """Return the customizations.locale.keyboard list + + :param recipe: The recipe + :type recipe: Recipe object + :returns: The keyboard layout string + :rtype: str + """ + if "customizations" not in recipe or \ + "locale" not in recipe["customizations"] or \ + "keyboard" not in recipe["customizations"]["locale"]: + return [] + return recipe["customizations"]["locale"]["keyboard"] + +
    +
    [docs]def firewall_cmd(line, settings): + """ Update the firewall line with the new ports and services + + :param line: The firewall ... line + :type line: str + :param settings: A dict with the list of services and ports to enable and disable + :type settings: dict + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + ks.readKickstartFromString(line) + + # Do not override firewall --disabled + if ks.handler.firewall.enabled != False and settings: + ks.handler.firewall.ports = sorted(set(settings["ports"] + ks.handler.firewall.ports)) + ks.handler.firewall.services = sorted(set(settings["enabled"] + ks.handler.firewall.services)) + ks.handler.firewall.remove_services = sorted(set(settings["disabled"] + ks.handler.firewall.remove_services)) + + # Converting back to a string includes a comment, return just the keyboard line + return str(ks.handler.firewall).splitlines()[-1] + +
    +
    [docs]def get_firewall_settings(recipe): + """Return the customizations.firewall settings + + :param recipe: The recipe + :type recipe: Recipe object + :returns: A dict of settings + :rtype: dict + """ + settings = {"ports": [], "enabled": [], "disabled": []} + + if "customizations" not in recipe or \ + "firewall" not in recipe["customizations"]: + return settings + + settings["ports"] = recipe["customizations"]["firewall"].get("ports", []) + + if "services" in recipe["customizations"]["firewall"]: + settings["enabled"] = recipe["customizations"]["firewall"]["services"].get("enabled", []) + settings["disabled"] = recipe["customizations"]["firewall"]["services"].get("disabled", []) + return settings + +
    +
    [docs]def services_cmd(line, settings): + """ Update the services line with additional services to enable/disable + + :param line: The services ... line + :type line: str + :param settings: A dict with the list of services to enable and disable + :type settings: dict + + Using pykickstart to process the line is the best way to make sure it + is parsed correctly, and re-assembled for inclusion into the final kickstart + """ + # Empty services and no additional settings, return an empty string + if not line and not settings["enabled"] and not settings["disabled"]: + return "" + + ks_version = makeVersion() + ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) + + # Allow passing in a 'default' so that the enable/disable may be applied to it, without + # parsing it and emitting a kickstart error message + if line != "services": + ks.readKickstartFromString(line) + + # Add to any existing services, removing any duplicates + ks.handler.services.enabled = sorted(set(settings["enabled"] + ks.handler.services.enabled)) + ks.handler.services.disabled = sorted(set(settings["disabled"] + ks.handler.services.disabled)) + + # Converting back to a string includes a comment, return just the keyboard line + return str(ks.handler.services).splitlines()[-1] + +
    +
    [docs]def get_services(recipe): + """Return the customizations.services settings + + :param recipe: The recipe + :type recipe: Recipe object + :returns: A dict of settings + :rtype: dict + """ + settings = {"enabled": [], "disabled": []} + + if "customizations" not in recipe or \ + "services" not in recipe["customizations"]: + return settings + + settings["enabled"] = sorted(recipe["customizations"]["services"].get("enabled", [])) + settings["disabled"] = sorted(recipe["customizations"]["services"].get("disabled", [])) + return settings + +
    +
    [docs]def get_default_services(recipe): + """Get the default string for services, based on recipe + :param recipe: The recipe + + :type recipe: Recipe object + :returns: string with "services" or "" + :rtype: str + + When no services have been selected we don't need to add anything to the kickstart + so return an empty string. Otherwise return "services" which will be updated with + the settings. + """ + services = get_services(recipe) + + if services["enabled"] or services["disabled"]: + return "services" + else: + return "" + +
    +
    [docs]def customize_ks_template(ks_template, recipe): + """ Customize the kickstart template and return it + + :param ks_template: The kickstart template + :type ks_template: str + :param recipe: + :type recipe: Recipe object + + Apply customizations to existing template commands, or add defaults for ones that are + missing and required. + + Apply customizations.kernel.append to the bootloader argument in the template. + Add bootloader line if it is missing. + + Add default timezone if needed. It does NOT replace an existing timezone entry + """ + # Commands to be modified [NEW-COMMAND-FUNC, NEW-VALUE, DEFAULT, REPLACE] + # The function is called with a kickstart command string and the value to replace + # The value is specific to the command, and is understood by the function + # The default is a complete kickstart command string, suitable for writing to the template + # If REPLACE is False it will not change an existing entry only add a missing one + commands = {"bootloader": [bootloader_append, + get_kernel_append(recipe), + 'bootloader --location=none', True], + "timezone": [timezone_cmd, + get_timezone_settings(recipe), + 'timezone UTC', False], + "lang": [lang_cmd, + get_languages(recipe), + 'lang en_US.UTF-8', True], + "keyboard": [keyboard_cmd, + get_keyboard_layout(recipe), + 'keyboard --xlayouts us --vckeymap us', True], + "firewall": [firewall_cmd, + get_firewall_settings(recipe), + 'firewall --enabled', True], + "services": [services_cmd, + get_services(recipe), + get_default_services(recipe), True] + } + found = {} + + output = StringIO() + for line in ks_template.splitlines(): + for cmd in sorted(commands.keys()): + (new_command, value, default, replace) = commands[cmd] + if line.startswith(cmd): + found[cmd] = True + if value and replace: + log.debug("Replacing %s with %s", cmd, value) + print(new_command(line, value), file=output) + else: + log.debug("Skipping %s", cmd) + print(line, file=output) + break + else: + # No matches, write the line as-is + print(line, file=output) + + # Write out defaults for the ones not found + # These must go FIRST because the template still needs to have the packages added + defaults = StringIO() + for cmd in sorted(commands.keys()): + if cmd in found: + continue + (new_command, value, default, _) = commands[cmd] + if value and default: + log.debug("Setting %s to use %s", cmd, value) + print(new_command(default, value), file=defaults) + elif default: + log.debug("Setting %s to %s", cmd, default) + print(default, file=defaults) + + return defaults.getvalue() + output.getvalue() + +
    +
    [docs]def write_ks_root(f, user): + """ Write kickstart root password and sshkey entry + + :param f: kickstart file object + :type f: open file object + :param user: A blueprint user dictionary + :type user: dict + + If the entry contains a ssh key, use sshkey to write it + If it contains password, use rootpw to set it + + root cannot be used with the user command. So only key and password are supported + for root. + """ + # ssh key uses the sshkey kickstart command + if "key" in user: + f.write('sshkey --user %s "%s"\n' % (user["name"], user["key"])) + + if "password" in user: + if any(user["password"].startswith(prefix) for prefix in ["$2b$", "$6$", "$5$"]): + log.debug("Detected pre-crypted password") + f.write('rootpw --iscrypted "%s"\n' % user["password"]) + else: + log.debug("Detected plaintext password") + f.write('rootpw --plaintext "%s"\n' % user["password"]) +
    [docs]def write_ks_user(f, user): """ Write kickstart user and sshkey entry @@ -187,9 +572,6 @@ All of the user fields are optional, except name, write out a kickstart user entry with whatever options are relevant. """ - if "name" not in user: - raise RuntimeError("user entry requires a name") - # ssh key uses the sshkey kickstart command if "key" in user: f.write('sshkey --user %s "%s"\n' % (user["name"], user["key"])) @@ -261,6 +643,10 @@ return customizations = recipe["customizations"] + # allow customizations to be incorrectly specified as [[customizations]] instead of [customizations] + if isinstance(customizations, list): + customizations = customizations[0] + if "hostname" in customizations: f.write("network --hostname=%s\n" % customizations["hostname"]) @@ -278,6 +664,14 @@ if "user" in customizations: # only name is required, everything else is optional for user in customizations["user"]: + if "name" not in user: + raise RuntimeError("user entry requires a name") + + # root is special, cannot use normal user command for it + if user["name"] == "root": + write_ks_root(f, user) + continue + write_ks_user(f, user) user_groups.append(user["name"]) @@ -287,6 +681,25 @@ write_ks_group(f, group) else: log.warning("Skipping group %s, already created by user", group["name"]) + +
    +
    [docs]def get_md_size(yum_path): + """Estimate the amount of space needed by anaconda + + Anaconda doesn't download the filelists or 'other' metadata, which can add + up to a significant difference, so exclude those from the calculation. + """ + try: + du_output = runcmd_output(["/usr/bin/du", + "--exclude", "*other*", + "--exclude", "*filelists*", + "-sb", + yum_path]) + return int(du_output.split()[0]) + except (ValueError, IndexError) as e: + log.error("Problem calculating metadata size from '%s': %s", du_output, str(e)) + return 0 +
    [docs]def start_build(cfg, yumlock, gitlock, branch, recipe_name, compose_type, test_mode=0): """ Start the build @@ -318,8 +731,9 @@ projects = sorted(set(module_nver+package_nver), key=lambda p: p[0].lower()) deps = [] try: - with yumlock.lock: - (installed_size, deps) = projects_depsolve_with_size(yumlock.yb, projects, recipe.group_names, with_core=False) + # This can possibly update repodata and reset the YumBase object. + with yumlock.lock_check: + (installed_size, anaconda_size, deps) = projects_depsolve_with_size(yumlock.yb, projects, recipe.group_names, with_core=False) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) @@ -328,7 +742,7 @@ ks_template_path = joinpaths(share_dir, "composer", compose_type) + ".ks" ks_template = open(ks_template_path, "r").read() - # How much space will the packages in the default template take? + # How much space will the packages in the selected template take? ks_version = makeVersion(RHEL7) ks = KickstartParser(ks_version, errorsAreFatal=False, missingIncludeIsFatal=False) ks.readKickstartFromString(ks_template+"\n%end\n") @@ -336,16 +750,29 @@ grps = [grp.name for grp in ks.handler.packages.groupList] try: with yumlock.lock: - (template_size, _) = projects_depsolve_with_size(yumlock.yb, pkgs, grps, - with_core=not ks.handler.packages.nocore) + (template_size, anaconda_tmpl_size, _) = projects_depsolve_with_size(yumlock.yb, pkgs, grps, with_core=not ks.handler.packages.nocore) except ProjectsError as e: log.error("start_build depsolve: %s", str(e)) raise RuntimeError("Problem depsolving %s: %s" % (recipe["name"], str(e))) - log.debug("installed_size = %d, template_size=%d", installed_size, template_size) - # Minimum LMC disk size is 1GiB, and anaconda bumps the estimated size up by 35% (which doesn't always work). - installed_size = max(1024**3, int((installed_size+template_size) * 1.4)) - log.debug("/ partition size = %d", installed_size) + # Anaconda also stores the metadata on the disk once it is partitioned, try to take this into account by + # adding the size of the lorax-composer metadata storage. + with yumlock.lock: + metadata_size = get_md_size(yumlock.yb.conf.installroot) + + # Anaconda estimates size differently, only taking into account installed size and adding 35% + # But we must make sure that our actual disk size is at least as big as the anaconda size, otherwise the install will fail + anaconda_minimum = int((anaconda_size+anaconda_tmpl_size) * 1.35) + + log.debug("anaconda_size = %d, anaconda_template_size=%d, anaconda_minimum=%d", anaconda_size, anaconda_tmpl_size, anaconda_minimum) + log.debug("installed_size = %d, template_size=%d, metadata_size=%d", installed_size, template_size, metadata_size) + + # Add 10% to the composer estimate + installed_size = int((installed_size+template_size+metadata_size) * 1.10) + + # Select the largest size for the partition + partition_size = max(1024**3, anaconda_minimum, installed_size) + log.debug("/ partition size = %d", partition_size) # Create the results directory build_id = str(uuid4()) @@ -397,14 +824,16 @@ f.write('clearpart --all\n') # Write the root partition and it's size in MB (rounded up) - f.write('part / --fstype="ext4" --size=%d\n' % ceil(installed_size / 1024**2)) + f.write('part / --size=%d\n' % ceil(installed_size / 1024**2)) - f.write(ks_template) + # Some customizations modify the template before writing it + f.write(customize_ks_template(ks_template, recipe)) for d in deps: f.write(dep_nevra(d)+"\n") f.write("%end\n") + # Other customizations can be appended to the kickstart add_customizations(f, recipe) # Setup the config to pass to novirt_install @@ -423,6 +852,7 @@ cfg_args["project"] = os_release.get("NAME") cfg_args["releasever"] = os_release.get("VERSION_ID") cfg_args["volid"] = "" + cfg_args["extra_boot_args"] = get_kernel_append(recipe) cfg_args.update({ "compression": "xz", @@ -565,6 +995,86 @@ "app_template": None, "app_file": None }, + "ami": {"make_iso": False, + "make_disk": True, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": False, + "qcow2_args": [], + "image_name": "disk.ami", + "fs_label": "", + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None, + }, + "vhd": {"make_iso": False, + "make_disk": True, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": True, + "qcow2_args": ["-O", "vpc", "-o", "subformat=fixed"], + "image_name": "disk.vhd", + "fs_label": "", + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None, + }, + "vmdk": {"make_iso": False, + "make_disk": True, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": True, + "qcow2_args": ["-O", "vmdk"], + "image_name": "disk.vmdk", + "fs_label": "", + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None, + }, + "openstack": {"make_iso": False, + "make_disk": True, + "make_fsimage": False, + "make_appliance": False, + "make_ami": False, + "make_tar": False, + "make_pxe_live": False, + "make_ostree_live": False, + "ostree": False, + "live_rootfs_keep_size": False, + "live_rootfs_size": 0, + "qcow2": True, + "qcow2_args": [], + "image_name": "disk.qcow2", + "fs_label": "", + "image_only": True, + "app_name": None, + "app_template": None, + "app_file": None, + }, } return _MAP[compose_type]
    @@ -623,7 +1133,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/config.html b/lorax-composer/_modules/pylorax/api/config.html index 6a03db3b..87a901a7 100644 --- a/lorax-composer/_modules/pylorax/api/config.html +++ b/lorax-composer/_modules/pylorax/api/config.html @@ -8,7 +8,7 @@ - pylorax.api.config — Lorax 19.7.22 documentation + pylorax.api.config — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -197,7 +197,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/crossdomain.html b/lorax-composer/_modules/pylorax/api/crossdomain.html index 0fa78996..6e2b5672 100644 --- a/lorax-composer/_modules/pylorax/api/crossdomain.html +++ b/lorax-composer/_modules/pylorax/api/crossdomain.html @@ -8,7 +8,7 @@ - pylorax.api.crossdomain — Lorax 19.7.22 documentation + pylorax.api.crossdomain — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -148,7 +148,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/projects.html b/lorax-composer/_modules/pylorax/api/projects.html index f65309f1..c758e8bb 100644 --- a/lorax-composer/_modules/pylorax/api/projects.html +++ b/lorax-composer/_modules/pylorax/api/projects.html @@ -8,7 +8,7 @@ - pylorax.api.projects — Lorax 19.7.22 documentation + pylorax.api.projects — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -70,12 +70,14 @@ import logging log = logging.getLogger("lorax-composer") -import os from ConfigParser import ConfigParser +import fnmatch from glob import glob +import os import time from yum.Errors import YumBaseError +from pylorax.api.bisect import insort_left TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" @@ -128,6 +130,32 @@ "homepage": yaps.url, "upstream_vcs": "UPSTREAM_VCS"} +
    +
    [docs]def yaps_to_build(yaps): + """Extract the build details from a hawkey.Package object + + :param yaps: Yum object with package details + :type yaps: YumAvailablePackageSqlite + :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ... + :rtype: dict + + metadata entries are hard-coded to {} + + Note that this only returns the build dict, it does not include the name, description, etc. + """ + return {"epoch": int(yaps.epoch), + "release": yaps.release, + "arch": yaps.arch, + "build_time": api_time(yaps.buildtime), + "changelog": api_changelog(yaps.returnChangelog()), + "build_config_ref": "BUILD_CONFIG_REF", + "build_env_ref": "BUILD_ENV_REF", + "metadata": {}, + "source": {"license": yaps.license, + "version": yaps.version, + "source_ref": "SOURCE_REF", + "metadata": {}}} +
    [docs]def yaps_to_project_info(yaps): """Extract the details from a YumAvailablePackageSqlite object @@ -139,25 +167,12 @@ metadata entries are hard-coded to {} """ - build = {"epoch": int(yaps.epoch), - "release": yaps.release, - "arch": yaps.arch, - "build_time": api_time(yaps.buildtime), - "changelog": api_changelog(yaps.returnChangelog()), - "build_config_ref": "BUILD_CONFIG_REF", - "build_env_ref": "BUILD_ENV_REF", - "metadata": {}, - "source": {"license": yaps.license, - "version": yaps.version, - "source_ref": "SOURCE_REF", - "metadata": {}}} - return {"name": yaps.name, "summary": yaps.summary, "description": yaps.description, "homepage": yaps.url, "upstream_vcs": "UPSTREAM_VCS", - "builds": [build]} + "builds": [yaps_to_build(yaps)]}
    [docs]def tm_to_dep(tm): @@ -175,17 +190,17 @@ "arch": tm.arch}
    -
    [docs]def yaps_to_module(yaps): +
    [docs]def proj_to_module(proj): """Extract the name from a YumAvailablePackageSqlite object - :param yaps: Yum object with package details - :type yaps: YumAvailablePackageSqlite + :param proj: Project details + :type proj: dict :returns: A dict with name, and group_type :rtype: dict group_type is hard-coded to "rpm" """ - return {"name": yaps.name, + return {"name": proj["name"], "group_type": "rpm"}
    @@ -220,8 +235,8 @@ raise ProjectsError("There was a problem listing projects: %s" % str(e)) finally: yb.closeRpmDB() - return sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower()) - + projs = _unique_dicts(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower()) + return sorted(projs, key=lambda p: p["name"].lower())
    [docs]def projects_info(yb, project_names): """Return details about specific projects @@ -239,9 +254,119 @@ raise ProjectsError("There was a problem with info for %s: %s" % (project_names, str(e))) finally: yb.closeRpmDB() - return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower()) - + return _unique_projs(ybl.available)
    +def _unique_projs(projs): + """Return a sorted list of projects with builds combined into one project entry + + :param yaps: Yum object with package details + :type yaps: YumAvailablePackageSqlite + :returns: List of project info dicts with yaps_to_project as well as epoch, version, release, etc. + :rtype: list of dicts + """ + # iterate over ybl.available + # - if ybl.name isn't in the results yet, add yaps_to_project_info in sorted position + # - if ybl.name is already in results, get its builds. If the build for the project is different + # in any way (version, arch, etc.) add it to the entry's builds list. If it is the same, + # skip it. + results = [] + results_names = {} + for p in projs: + if p.name.lower() not in results_names: + idx = insort_left(results, yaps_to_project_info(p), key=lambda p: p["name"].lower()) + results_names[p.name.lower()] = idx + else: + build = yaps_to_build(p) + if build not in results[results_names[p.name.lower()]]["builds"]: + results[results_names[p.name.lower()]]["builds"].append(build) + + return results + + +
    [docs]def filterVersionGlob(pkgs, version): + """Filter a list of yum package objects with a version glob + + :param pkgs: list of yum package objects + :type pkgs: list + :param version: version matching glob + :type version: str + + pkgs should be a list of all the versions of the *same* package. + Return the latest package that matches the 'version' glob. + """ + # Pick the version(s) matching the version glob + matches = [po for po in pkgs if fnmatch.fnmatchcase(po.version, version)] + if not matches: + raise RuntimeError("No package version matching %s" % version) + + # yum implements __cmd__ using verCMP so this will return the highest matching version + return max(matches) +
    +def _findPackageGlob(yb, pkg_name): + """Find the package(s) that match a name glob + + :param yb: yum base object + :type yb: YumBase + :param pkg_name: Name or fileglob of the name to find + :type pkg_name: str + :returns: list of yum package objects or empty list + """ + (exact, globbed, _unmatched) = yb.pkgSack.matchPackageNames([pkg_name]) + pkgs = exact + globbed + if pkgs: + return pkgs + + # Nothing matched, check rpmdb + pkgs = yb.rpmdb.returnPackages(patterns=[pkg_name], ignore_case=False) + if pkgs: + return pkgs + + # Nothing matched, find a matching dep + return yb.returnPackagesByDep(pkg_name) + + +def _depsolve(yb, projects, groups): + """Find the dependencies for a list of projects and groups + + :param yb: yum base object + :type yb: YumBase + :param projects: The projects and version globs to find the dependencies for + :type projects: List of tuples + :param groups: The groups to include in dependency solving + :type groups: List of str + :returns: A list of errors that were encountered while depsolving the packages + :rtype: list of strings + :raises: ProjectsError if there was a problem installing something + """ + # This resets the transaction + yb.closeRpmDB() + install_errors = [] + for name in groups: + yb.selectGroup(name, ["mandatory", "default"]) + + for name, version in projects: + if not version: + version = "*" + pattern = "%s %s" % (name, version) + + # yum.install's pattern matches the whole nevra, which can result in -* matching + # unexpected packages. So we need to implement our own version globbing. + # First get a list of packages, then filter that by the version + pkgs = _findPackageGlob(yb, name) + if not pkgs: + install_errors.append((name, "No package name matching %s" % name)) + continue + + try: + po = filterVersionGlob(pkgs, version) + log.debug("Chose %s as best match for %s", po.nevra, pattern) + + yb.install(po=po) + except (YumBaseError, RuntimeError) as e: + install_errors.append((pattern, str(e))) + + return install_errors +
    [docs]def projects_depsolve(yb, projects, groups): """Return the dependencies for a list of projects @@ -256,20 +381,7 @@ :raises: ProjectsError if there was a problem installing something """ try: - # This resets the transaction - yb.closeRpmDB() - install_errors = [] - for name in groups: - yb.selectGroup(name, ["mandatory", "default"]) - - for name, version in projects: - if not version: - version = "*" - pattern = "%s-%s" % (name, version) - try: - yb.install(pattern=pattern) - except YumBaseError as e: - install_errors.append((pattern, str(e))) + install_errors = _depsolve(yb, projects, groups) # Were there problems installing these packages? if install_errors: @@ -293,18 +405,26 @@ :type packages: list of TransactionMember objects :param block_size: The block size to use for rounding up file sizes. :type block_size: int - :returns: The estimated size of installed packages - :rtype: int + :returns: Tuple of the the estimated size needed, and the size anaconda will calculate + :rtype: tuple(int, int) Estimating actual requirements is difficult without the actual file sizes, which yum doesn't provide access to. So use the file count and block size to estimate a minimum size for each package. + + Anaconda only takes into account the installedsize of each package. It then fudges + this by 35% to make sure there is enough space. """ installed_size = 0 + anaconda_size = 0 for p in packages: installed_size += len(p.po.filelist) * block_size installed_size += p.po.installedsize - return installed_size + # anaconda only takes into account installedsize + anaconda_size += p.po.installedsize + # also count the RPM package size (yum cache) + installed_size += ((p.po.size / block_size) + 1) * block_size + return (installed_size, anaconda_size)
    [docs]def projects_depsolve_with_size(yb, projects, groups, with_core=True): """Return the dependencies and installed size for a list of projects @@ -315,25 +435,15 @@ :type projects: List of tuples :param groups: The groups to include in dependency solving :type groups: List of str - :returns: installed size and a list of NEVRA's of the project and its dependencies - :rtype: tuple of (int, list of dicts) + :returns: installed size, size estimated by anaconda, and a list of NEVRA's of the project and its dependencies + :rtype: tuple of (int, int, list of dicts) :raises: ProjectsError if there was a problem installing something + + The anaconda_size only includes the installed package size, not file block or cache estimation like + installed_size includes. """ try: - # This resets the transaction - yb.closeRpmDB() - install_errors = [] - for name in groups: - yb.selectGroup(name, ["mandatory", "default"]) - - for name, version in projects: - if not version: - version = "*" - pattern = "%s-%s" % (name, version) - try: - yb.install(pattern=pattern) - except YumBaseError as e: - install_errors.append((pattern, str(e))) + install_errors = _depsolve(yb, projects, groups) # Were there problems installing these packages? if install_errors: @@ -345,13 +455,13 @@ if rc not in [0, 1, 2]: raise ProjectsError("There was a problem depsolving %s: %s" % (projects, msg)) yb.tsInfo.makelists() - installed_size = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled) + (installed_size, anaconda_size) = estimate_size(yb.tsInfo.installed + yb.tsInfo.depinstalled) deps = sorted(map(tm_to_dep, yb.tsInfo.installed + yb.tsInfo.depinstalled), key=lambda p: p["name"].lower()) except YumBaseError as e: raise ProjectsError("There was a problem depsolving %s: %s" % (projects, str(e))) finally: yb.closeRpmDB() - return (installed_size, deps) + return (installed_size, anaconda_size, deps)
    [docs]def modules_list(yb, module_names): """Return a list of modules @@ -368,15 +478,30 @@ Modules don't exist in RHEL7 so this only returns projects and sets the type to "rpm" """ - try: - ybl = yb.doPackageLists(pkgnarrow="available", patterns=module_names, showdups=False) - except YumBaseError as e: - raise ProjectsError("There was a problem listing modules: %s" % str(e)) - finally: - yb.closeRpmDB() - return sorted(map(yaps_to_module, ybl.available), key=lambda p: p["name"].lower()) - + projs = _unique_dicts(projects_info(yb, module_names), key=lambda p: p["name"].lower()) + return list(map(proj_to_module, projs))
    +def _unique_dicts(lst, key): + """Return a new list of dicts, only including one match of key(d) + + :param lst: list of dicts + :type lst: list + :param key: key function to match lst entries + :type key: function + :returns: list of the unique lst entries + :rtype: list + + Uses key(d) to test for duplicates in the returned list, creating a + list of unique return values. + """ + result = [] + result_keys = [] + for d in lst: + if key(d) not in result_keys: + result.append(d) + result_keys.append(key(d)) + return result +
    [docs]def modules_info(yb, module_names): """Return details about a module, including dependencies @@ -648,7 +773,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/queue.html b/lorax-composer/_modules/pylorax/api/queue.html index 4b59fa43..67218f88 100644 --- a/lorax-composer/_modules/pylorax/api/queue.html +++ b/lorax-composer/_modules/pylorax/api/queue.html @@ -8,7 +8,7 @@ - pylorax.api.queue — Lorax 19.7.22 documentation + pylorax.api.queue — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -88,6 +88,47 @@ from pylorax.creator import run_creator from pylorax.sysutils import joinpaths +
    [docs]def check_queues(cfg): + """Check to make sure the new and run queue symlinks are correct + + :param cfg: Configuration settings + :type cfg: DataHolder + + Also check all of the existing results and make sure any with WAITING + set in STATUS have a symlink in queue/new/ + """ + # Remove broken symlinks from the new and run queues + queue_symlinks = glob(joinpaths(cfg.composer_dir, "queue/new/*")) + \ + glob(joinpaths(cfg.composer_dir, "queue/run/*")) + for link in queue_symlinks: + if not os.path.isdir(os.path.realpath(link)): + log.info("Removing broken symlink %s", link) + os.unlink(link) + + # Write FAILED to the STATUS of any run queue symlinks and remove them + for link in glob(joinpaths(cfg.composer_dir, "queue/run/*")): + log.info("Setting build %s to FAILED, and removing symlink from queue/run/", os.path.basename(link)) + open(joinpaths(link, "STATUS"), "w").write("FAILED\n") + os.unlink(link) + + # Check results STATUS messages + # - If STATUS is missing, set it to FAILED + # - RUNNING should be changed to FAILED + # - WAITING should have a symlink in the new queue + for link in glob(joinpaths(cfg.composer_dir, "results/*")): + if not os.path.exists(joinpaths(link, "STATUS")): + open(joinpaths(link, "STATUS"), "w").write("FAILED\n") + continue + + status = open(joinpaths(link, "STATUS")).read().strip() + if status == "RUNNING": + log.info("Setting build %s to FAILED", os.path.basename(link)) + open(joinpaths(link, "STATUS"), "w").write("FAILED\n") + elif status == "WAITING": + if not os.path.islink(joinpaths(cfg.composer_dir, "queue/new/", os.path.basename(link))): + log.info("Creating missing symlink to new build %s", os.path.basename(link)) + os.symlink(link, joinpaths(cfg.composer_dir, "queue/new/", os.path.basename(link))) +
    [docs]def start_queue_monitor(cfg, uid, gid): """Start the queue monitor as a mp process @@ -121,7 +162,7 @@ compose is finished) the symlink will be moved into ./queue/run/ and a STATUS file will be created in the results directory. - STATUS can contain one of: RUNNING, FINISHED, FAILED + STATUS can contain one of: WAITING, RUNNING, FINISHED, FAILED If the system is restarted while a compose is running it will move any old symlinks from ./queue/run/ to ./queue/new/ and rerun them. @@ -130,13 +171,7 @@ """Sort the queue entries by their mtime, not their names""" return os.stat(joinpaths(cfg.composer_dir, "queue/new", uuid)).st_mtime - # Move any symlinks in the run queue back to the new queue - for link in os.listdir(joinpaths(cfg.composer_dir, "queue/run")): - src = joinpaths(cfg.composer_dir, "queue/run", link) - dst = joinpaths(cfg.composer_dir, "queue/new", link) - os.rename(src, dst) - log.debug("Moved unfinished compose %s back to new state", src) - + check_queues(cfg) while True: uuids = sorted(os.listdir(joinpaths(cfg.composer_dir, "queue/new")), key=queue_sort) @@ -259,7 +294,7 @@ else: open(joinpaths(results_dir, install_cfg.image_name), "w").write("TEST IMAGE") else: - run_creator(install_cfg, callback_func=cancel_build) + run_creator(install_cfg, cancel_func=cancel_build) # Extract the results of the compose into results_dir and cleanup the compose directory move_compose_results(install_cfg, results_dir) @@ -442,11 +477,16 @@ Only call this if the build status is WAITING or RUNNING """ + cancel_path = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid, "CANCEL") + if os.path.exists(cancel_path): + log.info("Cancel has already been requested for %s", uuid) + return False + # This status can change (and probably will) while it is in the middle of doing this: # It can move from WAITING -> RUNNING or it can move from RUNNING -> FINISHED|FAILED # If it is in WAITING remove the symlink and then check to make sure it didn't show up - # in RUNNING + # in the run queue queue_dir = joinpaths(cfg.get("composer", "lib_dir"), "queue") uuid_new = joinpaths(queue_dir, "new", uuid) if os.path.exists(uuid_new): @@ -457,19 +497,24 @@ pass uuid_run = joinpaths(queue_dir, "run", uuid) if not os.path.exists(uuid_run): - # Successfully removed it before the build started - return uuid_delete(cfg, uuid) + # Make sure the build is still in the waiting state + status = uuid_status(cfg, uuid) + if status is None or status["queue_status"] == "WAITING": + # Successfully removed it before the build started + return uuid_delete(cfg, uuid) - # Tell the build to stop running - cancel_path = joinpaths(cfg.get("composer", "lib_dir"), "results", uuid, "CANCEL") + # At this point the build has probably started. Write to the CANCEL file. open(cancel_path, "w").write("\n") - # Wait for status to move to FAILED + # Wait for status to move to FAILED or FINISHED started = time.time() while True: status = uuid_status(cfg, uuid) if status is None or status["queue_status"] == "FAILED": break + elif status is not None and status["queue_status"] == "FINISHED": + # The build finished successfully, no point in deleting it now + return False # Is this taking too long? Exit anyway and try to cleanup. if time.time() > started + (10 * 60): @@ -710,7 +755,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/recipes.html b/lorax-composer/_modules/pylorax/api/recipes.html index 9076c9f3..0026bdeb 100644 --- a/lorax-composer/_modules/pylorax/api/recipes.html +++ b/lorax-composer/_modules/pylorax/api/recipes.html @@ -8,7 +8,7 @@ - pylorax.api.recipes — Lorax 19.7.22 documentation + pylorax.api.recipes — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -52,7 +52,7 @@

    Source code for pylorax.api.recipes

     #
    -# Copyright (C) 2017  Red Hat, Inc.
    +# Copyright (C) 2017-2019  Red Hat, Inc.
     #
     # This program is free software; you can redistribute it and/or modify
     # it under the terms of the GNU General Public License as published by
    @@ -287,6 +287,12 @@
             description = recipe_dict["description"]
             version = recipe_dict.get("version", None)
             customizations = recipe_dict.get("customizations", None)
    +
    +        # [customizations] was incorrectly documented at first, so we have to support using it
    +        # as [[customizations]] by grabbing the first element.
    +        if isinstance(customizations, list):
    +            customizations = customizations[0]
    +
         except KeyError as e:
             raise RecipeError("There was a problem parsing the recipe: %s" % str(e))
     
    @@ -461,6 +467,9 @@
         If no commit is passed the master:filename is returned, otherwise it will be
         commit:filename
         """
    +    if not repo_file_exists(repo, branch, recipe_filename(recipe_name)):
    +        raise RecipeFileError("Unknown blueprint")
    +
         (_, recipe_toml) = read_commit(repo, branch, recipe_filename(recipe_name), commit)
         return recipe_from_toml(recipe_toml)
     
    @@ -684,6 +693,9 @@ Uses tag_file_commit() """ + if not repo_file_exists(repo, branch, recipe_filename(recipe_name)): + raise RecipeFileError("Unknown blueprint") + return tag_file_commit(repo, branch, recipe_filename(recipe_name))
    [docs]def tag_file_commit(repo, branch, filename): @@ -893,25 +905,78 @@ diff = Git.Diff.new_tree_to_tree(repo, parent.get_tree(), tree, diff_opts) return diff.get_num_deltas() > 0
    +
    [docs]def find_field_value(field, value, lst): + """Find a field matching value in the list of dicts. + + :param field: field to search for + :type field: str + :param value: value to match in the field + :type value: str + :param lst: List of dict's with field + :type lst: list of dict + :returns: First dict with matching field:value, or None + :rtype: dict or None + + Used to return a specific entry from a list that looks like this: + + [{"name": "one", "attr": "green"}, ...] + + find_field_value("name", "one", lst) will return the matching dict. + """ + for d in lst: + if d.get(field) and d.get(field) == value: + return d + return None +
    [docs]def find_name(name, lst): """Find the dict matching the name in a list and return it. :param name: Name to search for :type name: str :param lst: List of dict's with "name" field + :type lst: list of dict :returns: First dict with matching name, or None :rtype: dict or None + + This is just a wrapper for find_field_value with field set to "name" """ - for e in lst: - if e["name"] == name: - return e - return None + return find_field_value("name", name, lst)
    -
    [docs]def diff_items(title, old_items, new_items): +
    [docs]def find_recipe_obj(path, recipe, default=None): + """Find a recipe object + + :param path: A list of dict field names + :type path: list of str + :param recipe: The recipe to search + :type recipe: Recipe + :param default: The value to return if it is not found + :type default: Any + + Return the object found by applying the path to the dicts in the recipe, or + return the default if it doesn't exist. + + eg. {"customizations": {"hostname": "foo", "users": [...]}} + + find_recipe_obj(["customizations", "hostname"], recipe, "") + """ + o = recipe + try: + for p in path: + if not o.get(p): + return default + o = o.get(p) + except AttributeError: + return default + + return o +
    +
    [docs]def diff_lists(title, field, old_items, new_items): """Return the differences between two lists of dicts. :param title: Title of the entry :type title: str + :param field: Field to use as the key for comparisons + :type field: str :param old_items: List of item dicts with "name" field :type old_items: list(dict) :param new_items: List of item dicts with "name" field @@ -920,34 +985,79 @@ :rtype: list(dict) """ diffs = [] - old_names = set(m["name"] for m in old_items) - new_names = set(m["name"] for m in new_items) + old_fields= set(m[field] for m in old_items) + new_fields= set(m[field] for m in new_items) - added_items = new_names.difference(old_names) + added_items = new_fields.difference(old_fields) added_items = sorted(added_items, key=lambda n: n.lower()) - removed_items = old_names.difference(new_names) + removed_items = old_fields.difference(new_fields) removed_items = sorted(removed_items, key=lambda n: n.lower()) - same_items = old_names.intersection(new_names) + same_items = old_fields.intersection(new_fields) same_items = sorted(same_items, key=lambda n: n.lower()) - for name in added_items: + for v in added_items: diffs.append({"old":None, - "new":{title:find_name(name, new_items)}}) + "new":{title:find_field_value(field, v, new_items)}}) - for name in removed_items: - diffs.append({"old":{title:find_name(name, old_items)}, + for v in removed_items: + diffs.append({"old":{title:find_field_value(field, v, old_items)}, "new":None}) - for name in same_items: - old_item = find_name(name, old_items) - new_item = find_name(name, new_items) + for v in same_items: + old_item = find_field_value(field, v, old_items) + new_item = find_field_value(field, v, new_items) if old_item != new_item: diffs.append({"old":{title:old_item}, "new":{title:new_item}}) return diffs +
    +
    [docs]def customizations_diff(old_recipe, new_recipe): + """Diff the customizations sections from two versions of a recipe + """ + diffs = [] + old_keys = set(old_recipe.get("customizations", {}).keys()) + new_keys = set(new_recipe.get("customizations", {}).keys()) + + added_keys = new_keys.difference(old_keys) + added_keys = sorted(added_keys, key=lambda n: n.lower()) + + removed_keys = old_keys.difference(new_keys) + removed_keys = sorted(removed_keys, key=lambda n: n.lower()) + + same_keys = old_keys.intersection(new_keys) + same_keys = sorted(same_keys, key=lambda n: n.lower()) + + for v in added_keys: + diffs.append({"old": None, + "new": {"Customizations."+v: new_recipe["customizations"][v]}}) + + for v in removed_keys: + diffs.append({"old": {"Customizations."+v: old_recipe["customizations"][v]}, + "new": None}) + + for v in same_keys: + if new_recipe["customizations"][v] == old_recipe["customizations"][v]: + continue + + if type(new_recipe["customizations"][v]) == type([]): + # Lists of dicts need to use diff_lists + # sshkey uses 'user', user and group use 'name' + if "user" in new_recipe["customizations"][v][0]: + field_name = "user" + elif "name" in new_recipe["customizations"][v][0]: + field_name = "name" + else: + raise RuntimeError("%s list has unrecognized key, not 'name' or 'user'" % "customizations."+v) + + diffs.extend(diff_lists("Customizations."+v, field_name, old_recipe["customizations"][v], new_recipe["customizations"][v])) + else: + diffs.append({"old": {"Customizations."+v: old_recipe["customizations"][v]}, + "new": {"Customizations."+v: new_recipe["customizations"][v]}}) + + return diffs
    [docs]def recipe_diff(old_recipe, new_recipe): @@ -968,11 +1078,38 @@ diffs.append({"old":{element.title():old_recipe[element]}, "new":{element.title():new_recipe[element]}}) - diffs.extend(diff_items("Module", old_recipe["modules"], new_recipe["modules"])) - diffs.extend(diff_items("Package", old_recipe["packages"], new_recipe["packages"])) - diffs.extend(diff_items("Group", old_recipe["groups"], new_recipe["groups"])) + # These lists always exist + diffs.extend(diff_lists("Module", "name", old_recipe["modules"], new_recipe["modules"])) + diffs.extend(diff_lists("Package", "name", old_recipe["packages"], new_recipe["packages"])) + diffs.extend(diff_lists("Group", "name", old_recipe["groups"], new_recipe["groups"])) - return diffs
    + # The customizations section can contain a number of different types + diffs.extend(customizations_diff(old_recipe, new_recipe)) + + # repos contains keys that are lists (eg. [[repos.git]]) + diffs.extend(diff_lists("Repos.git", "rpmname", + find_recipe_obj(["repos", "git"], old_recipe, []), + find_recipe_obj(["repos", "git"], new_recipe, []))) + + return diffs +
    +
    [docs]def repo_file_exists(repo, branch, filename): + """Return True if the filename exists on the branch + + :param repo: Open repository + :type repo: Git.Repository + :param branch: Branch name + :type branch: str + :param filename: Filename to check + :type filename: str + :returns: True if the filename exists on the HEAD of the branch, False otherwise. + :rtype: bool + """ + commit = head_commit(repo, branch).get_id().to_string() + commit_id = Git.OId.new_from_string(commit) + commit_obj = repo.lookup(commit_id, Git.Commit) + tree = commit_obj.get_tree() + return tree.get_by_name(filename) is not None
    @@ -1006,7 +1143,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/server.html b/lorax-composer/_modules/pylorax/api/server.html index 5fff9496..5212cbc8 100644 --- a/lorax-composer/_modules/pylorax/api/server.html +++ b/lorax-composer/_modules/pylorax/api/server.html @@ -8,7 +8,7 @@ - pylorax.api.server — Lorax 19.7.22 documentation + pylorax.api.server — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -81,7 +81,6 @@ from pylorax.sysutils import joinpaths GitLock = namedtuple("GitLock", ["repo", "lock", "dir"]) -YumLock = namedtuple("YumLock", ["yb", "lock"]) server = Flask(__name__) @@ -167,7 +166,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/timestamp.html b/lorax-composer/_modules/pylorax/api/timestamp.html index ec5ac58d..e53a66df 100644 --- a/lorax-composer/_modules/pylorax/api/timestamp.html +++ b/lorax-composer/_modules/pylorax/api/timestamp.html @@ -8,7 +8,7 @@ - pylorax.api.timestamp — Lorax 19.7.22 documentation + pylorax.api.timestamp — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -135,7 +135,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/v0.html b/lorax-composer/_modules/pylorax/api/v0.html index 2e3e3a22..119c4952 100644 --- a/lorax-composer/_modules/pylorax/api/v0.html +++ b/lorax-composer/_modules/pylorax/api/v0.html @@ -8,7 +8,7 @@ - pylorax.api.v0 — Lorax 19.7.22 documentation + pylorax.api.v0 — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -116,15 +116,23 @@ "kubernetes" ], "total": 6 } -`/api/v0/blueprints/info/<blueprint_names>` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`/api/v0/blueprints/info/<blueprint_names>[?format=<json|toml>]` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Return the JSON representation of the blueprint. This includes 3 top level objects. `changes` which lists whether or not the workspace is different from the most recent commit. `blueprints` which lists the JSON representation of the blueprint, and `errors` which will list any errors, like non-existant blueprints. - Example:: + By default the response is JSON, but if `?format=toml` is included in the URL's + arguments it will return the response as the blueprint's raw TOML content. + *Unless* there is an error which will only return a 400 and a standard error + `Status Response`_. + + If there is an error when JSON is requested the successful blueprints and the + errors will both be returned. + + Example of json response:: { "changes": [ @@ -246,7 +254,8 @@ Delete a blueprint. The blueprint is deleted from the branch, and will no longer be listed by the `list` route. A blueprint can be undeleted using the `undo` route - to revert to a previous commit. + to revert to a previous commit. This will also delete the workspace copy of the + blueprint. The response will be a status response with `status` set to true, or an error response with it set to false and an error message included. @@ -1045,7 +1054,7 @@ from pylorax.api.queue import uuid_tar, uuid_image, uuid_cancel, uuid_log from pylorax.api.recipes import RecipeError, list_branch_files, read_recipe_commit, recipe_filename, list_commits from pylorax.api.recipes import recipe_from_dict, recipe_from_toml, commit_recipe, delete_recipe, revert_recipe -from pylorax.api.recipes import tag_recipe_commit, recipe_diff +from pylorax.api.recipes import tag_recipe_commit, recipe_diff, RecipeFileError from pylorax.api.regexes import VALID_API_STRING from pylorax.api.workspace import workspace_read, workspace_write, workspace_delete from pylorax.api.yumbase import update_metadata @@ -1067,12 +1076,21 @@ return iterable[offset:][:limit]
    [docs]def blueprint_exists(api, branch, blueprint_name): + """Return True if the blueprint exists + + :param api: flask object + :type api: Flask + :param branch: Branch name + :type branch: str + :param recipe_name: Recipe name to read + :type recipe_name: str + """ try: with api.config["GITLOCK"].lock: read_recipe_commit(api.config["GITLOCK"].repo, branch, blueprint_name) return True - except RecipeError: + except (RecipeError, RecipeFileError): return False
    [docs]def v0_api(api): @@ -1131,6 +1149,10 @@ try: with api.config["GITLOCK"].lock: git_blueprint = read_recipe_commit(api.config["GITLOCK"].repo, branch, blueprint_name) + except RecipeFileError as e: + # Adding an exception would be redundant, skip it + git_blueprint = None + log.error("(v0_blueprints_info) %s", str(e)) except Exception as e: git_blueprint = None exceptions.append(str(e)) @@ -1157,8 +1179,12 @@ blueprints = sorted(blueprints, key=lambda r: r["name"].lower()) if out_fmt == "toml": - # With TOML output we just want to dump the raw blueprint, skipping the rest. - return "\n\n".join([r.toml() for r in blueprints]) + if errors: + # If there are errors they need to be reported, use JSON and 400 for this + return jsonify(status=False, errors=errors), 400 + else: + # With TOML output we just want to dump the raw blueprint, skipping the rest. + return "\n\n".join([r.toml() for r in blueprints]) else: return jsonify(changes=changes, blueprints=blueprints, errors=errors) @@ -1185,20 +1211,19 @@ errors = [] for blueprint_name in [n.strip() for n in blueprint_names.split(",")]: filename = recipe_filename(blueprint_name) - - if not blueprint_exists(api, branch, blueprint_name): - errors.append({"id": UNKNOWN_BLUEPRINT, "msg": "Unknown blueprint name: %s" % blueprint_name}) - continue - try: with api.config["GITLOCK"].lock: commits = list_commits(api.config["GITLOCK"].repo, branch, filename) - limited_commits = take_limits(list_commits(api.config["GITLOCK"].repo, branch, filename), offset, limit) except Exception as e: errors.append({"id": BLUEPRINTS_ERROR, "msg": "%s: %s" % (blueprint_name, str(e))}) log.error("(v0_blueprints_changes) %s", str(e)) else: - blueprints.append({"name":blueprint_name, "changes":limited_commits, "total":len(commits)}) + if commits: + limited_commits = take_limits(commits, offset, limit) + blueprints.append({"name":blueprint_name, "changes":limited_commits, "total":len(commits)}) + else: + # no commits means there is no blueprint in the branch + errors.append({"id": UNKNOWN_BLUEPRINT, "msg": "%s" % blueprint_name}) blueprints = sorted(blueprints, key=lambda r: r["name"].lower()) @@ -1248,6 +1273,7 @@ try: with api.config["GITLOCK"].lock: + workspace_delete(api.config["GITLOCK"].repo, branch, blueprint_name) delete_recipe(api.config["GITLOCK"].repo, branch, blueprint_name) except Exception as e: log.error("(v0_blueprints_delete) %s", str(e)) @@ -1346,6 +1372,9 @@ try: with api.config["GITLOCK"].lock: tag_recipe_commit(api.config["GITLOCK"].repo, branch, blueprint_name) + except RecipeFileError as e: + log.error("(v0_blueprints_tag) %s", str(e)) + return jsonify(status=False, errors=[{"id": UNKNOWN_BLUEPRINT, "msg": str(e)}]), 400 except Exception as e: log.error("(v0_blueprints_tag) %s", str(e)) return jsonify(status=False, errors=[{"id": BLUEPRINTS_ERROR, "msg": str(e)}]), 400 @@ -1370,6 +1399,9 @@ if VALID_API_STRING.match(branch) is None: return jsonify(status=False, errors=[{"id": INVALID_CHARS, "msg": "Invalid characters in branch argument"}]), 400 + if not blueprint_exists(api, branch, blueprint_name): + return jsonify(status=False, errors=[{"id": UNKNOWN_BLUEPRINT, "msg": "Unknown blueprint name: %s" % blueprint_name}]) + try: if from_commit == "NEWEST": with api.config["GITLOCK"].lock: @@ -1436,6 +1468,9 @@ try: with api.config["GITLOCK"].lock: blueprint = read_recipe_commit(api.config["GITLOCK"].repo, branch, blueprint_name) + except RecipeFileError as e: + # adding an error here would be redundant, skip it + log.error("(v0_blueprints_freeze) %s", str(e)) except Exception as e: errors.append({"id": BLUEPRINTS_ERROR, "msg": "%s: %s" % (blueprint_name, str(e))}) log.error("(v0_blueprints_freeze) %s", str(e)) @@ -1496,6 +1531,9 @@ try: with api.config["GITLOCK"].lock: blueprint = read_recipe_commit(api.config["GITLOCK"].repo, branch, blueprint_name) + except RecipeFileError as e: + # adding an error here would be redundant, skip it + log.error("(v0_blueprints_depsolve) %s", str(e)) except Exception as e: errors.append({"id": BLUEPRINTS_ERROR, "msg": "%s: %s" % (blueprint_name, str(e))}) log.error("(v0_blueprints_depsolve) %s", str(e)) @@ -1834,6 +1872,9 @@ if VALID_API_STRING.match(blueprint_name) is None: errors.append({"id": INVALID_CHARS, "msg": "Invalid characters in API path"}) + if not blueprint_exists(api, branch, blueprint_name): + errors.append({"id": UNKNOWN_BLUEPRINT, "msg": "Unknown blueprint name: %s" % blueprint_name}) + if errors: return jsonify(status=False, errors=errors), 400 @@ -2130,7 +2171,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/workspace.html b/lorax-composer/_modules/pylorax/api/workspace.html index 4e3b4d23..812063ed 100644 --- a/lorax-composer/_modules/pylorax/api/workspace.html +++ b/lorax-composer/_modules/pylorax/api/workspace.html @@ -8,7 +8,7 @@ - pylorax.api.workspace — Lorax 19.7.22 documentation + pylorax.api.workspace — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -183,7 +183,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/api/yumbase.html b/lorax-composer/_modules/pylorax/api/yumbase.html index 3b4f95bb..6032ab06 100644 --- a/lorax-composer/_modules/pylorax/api/yumbase.html +++ b/lorax-composer/_modules/pylorax/api/yumbase.html @@ -8,7 +8,7 @@ - pylorax.api.yumbase — Lorax 19.7.22 documentation + pylorax.api.yumbase — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • @@ -76,6 +76,8 @@ from fnmatch import fnmatchcase from glob import glob import os +from threading import Lock +import time import yum from yum.Errors import YumBaseError @@ -84,6 +86,74 @@ from pylorax.sysutils import joinpaths +
    [docs]class YumLock(object): + """Hold the YumBase object and a Lock to control access to it. + + self.yb is a property that returns the YumBase object, but it *may* change + from one call to the next if the upstream repositories have changed. + """ + def __init__(self, conf, expire_secs=6*60*60): + self._conf = conf + self._lock = Lock() + self.yb = get_base_object(self._conf) + self._expire_secs = expire_secs + self._expire_time = time.time() + self._expire_secs + + @property +
    [docs] def lock(self): + """Check for repo updates (using expiration time) and return the lock + + If the repository has been updated, tear down the old YumBase and + create a new one. This is the only way to force yum to use the new + metadata. + """ + if time.time() > self._expire_time: + return self.lock_check + return self._lock +
    + @property +
    [docs] def lock_check(self): + """Force a check for repo updates and return the lock + + If the repository has been updated, tear down the old YumBase and + create a new one. This is the only way to force yum to use the new + metadata. + + Use this method sparingly, it removes the repodata and downloads a new copy every time. + """ + self._expire_time = time.time() + self._expire_secs + if self._haveReposChanged(): + self._destroyYb() + self.yb = get_base_object(self._conf) + return self._lock +
    + def _destroyYb(self): + # Do our best to get yum to let go of all the things... + self.yb.pkgSack.dropCachedData() + for s in self.yb.pkgSack.sacks.values(): + s.close() + del s + del self.yb.pkgSack + self.yb.closeRpmDB() + del self.yb.tsInfo + del self.yb.ts + self.yb.close() + del self.yb + + def _haveReposChanged(self): + """Return True if the repo has new metadata""" + # This is a total kludge, yum doesn't really expect to deal with things changing while the + # object is is use. + try: + before = [(r.id, r.repoXML.checksums["sha256"]) for r in sorted(self.yb.repos.listEnabled())] + for r in sorted(self.yb.repos.listEnabled()): + r.metadata_expire = 0 + del r.repoXML + after = [(r.id, r.repoXML.checksums["sha256"]) for r in sorted(self.yb.repos.listEnabled())] + return before != after + except Exception: + return False +
    [docs]def get_base_object(conf): """Get the Yum object with settings from the config file @@ -156,6 +226,32 @@ if any(map(lambda pattern: fnmatchcase(name, pattern), enabled_repos)): # pylint: disable=cell-var-from-loop yb.getReposFromConfigFile(repo_file) + # Remove any duplicate repo entries. These can cause problems with Anaconda, which will fail + # with space problems. + repos = list(r.id for r in yb.repos.listEnabled()) + seen = {"baseurl": [], "mirrorlist": [], "metalink": []} + for source_name in repos: + remove = False + repo = yb.repos.getRepo(source_name) + if repo.baseurl: + if repo.baseurl[0] in seen["baseurl"]: + log.info("Removing duplicate repo: %s baseurl=%s", source_name, repo.baseurl[0]) + remove = True + else: + seen["baseurl"].append(repo.baseurl[0]) + elif repo.mirrorlist: + if repo.mirrorlist in seen["mirrorlist"]: + log.info("Removing duplicate repo: %s mirrorlist=%s", source_name, repo.mirrorlist) + remove = True + else: + seen["mirrorlist"].append(repo.mirrorlist) + + if remove: + yb.repos.delete(source_name) + + # delete doesn't remove it from the cache used by listEnabled so we have to force it + yb.repos._cache_enabled_repos = None + # Update the metadata from the enabled repos to speed up later operations log.info("Updating yum repository metadata") update_metadata(yb) @@ -213,7 +309,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • pylorax.api »
  • diff --git a/lorax-composer/_modules/pylorax/base.html b/lorax-composer/_modules/pylorax/base.html index 620aaadc..bf04e5d3 100644 --- a/lorax-composer/_modules/pylorax/base.html +++ b/lorax-composer/_modules/pylorax/base.html @@ -8,7 +8,7 @@ - pylorax.base — Lorax 19.7.22 documentation + pylorax.base — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -150,7 +150,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/buildstamp.html b/lorax-composer/_modules/pylorax/buildstamp.html index ca9d07d2..73d58b5a 100644 --- a/lorax-composer/_modules/pylorax/buildstamp.html +++ b/lorax-composer/_modules/pylorax/buildstamp.html @@ -8,7 +8,7 @@ - pylorax.buildstamp — Lorax 19.7.22 documentation + pylorax.buildstamp — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -141,7 +141,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/creator.html b/lorax-composer/_modules/pylorax/creator.html index e4e40cc2..adad3524 100644 --- a/lorax-composer/_modules/pylorax/creator.html +++ b/lorax-composer/_modules/pylorax/creator.html @@ -8,7 +8,7 @@ - pylorax.creator — Lorax 19.7.22 documentation + pylorax.creator — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -97,7 +97,7 @@ RUNTIME = "images/install.img" -# Default parameters for rebuilding initramfs, override with --dracut-args +# Default parameters for rebuilding initramfs, override with --dracut-arg DRACUT_DEFAULT = ["--xz", "--add", "livenet dmsquash-live convertfs pollcdrom", "--omit", "plymouth", "--no-hostonly", "--no-early-microcode"] @@ -393,7 +393,8 @@ tb = TreeBuilder(product=product, arch=arch, domacboot=opts.domacboot, inroot=mount_dir, outroot=work_dir, runtime=RUNTIME, isolabel=isolabel, - templatedir=joinpaths(opts.lorax_templates,"live/")) + templatedir=joinpaths(opts.lorax_templates,"live/"), + extra_boot_args=opts.extra_boot_args) log.info( "Rebuilding initrds" ) if not opts.dracut_args: dracut_args = DRACUT_DEFAULT @@ -457,7 +458,7 @@ remove(joinpaths(work_dir, "runtime"))
    -
    [docs]def make_image(opts, ks, callback_func=None): +
    [docs]def make_image(opts, ks, cancel_func=None): """ Install to an image @@ -475,12 +476,12 @@ try: if opts.no_virt: - novirt_install(opts, disk_img, disk_size, ks.handler.method.url, callback_func=callback_func) + novirt_install(opts, disk_img, disk_size, ks.handler.method.url, cancel_func=cancel_func) else: install_log = os.path.abspath(os.path.dirname(opts.logfile))+"/virt-install.log" log.info("install_log = %s", install_log) - virt_install(opts, install_log, disk_img, disk_size) + virt_install(opts, install_log, disk_img, disk_size, cancel_func=cancel_func) except InstallError as e: log.error("Install failed: %s", e) if not opts.keep_image: @@ -541,7 +542,7 @@ return work_dir
    -
    [docs]def run_creator(opts, callback_func=None): +
    [docs]def run_creator(opts, cancel_func=None): """Run the image creator process :param opts: Commandline options to control the process @@ -589,7 +590,10 @@ # Make the image. Output of this is either a partitioned disk image or a fsimage # Can also fail with InstallError - disk_img = make_image(opts, ks, callback_func=callback_func) + disk_img = make_image(opts, ks, cancel_func=cancel_func) + + if cancel_func and cancel_func(): + raise RuntimeError("image creation canceled") # Only create the disk image, return that now if opts.image_only: @@ -604,6 +608,10 @@ disk_img = opts.fs_image or disk_img make_squashfs(disk_img, work_dir) + + if cancel_func and cancel_func(): + raise RuntimeError("ISO creation canceled") + with Mount(disk_img, opts="loop") as mount_dir: result_dir = make_livecd(opts, mount_dir, work_dir) else: @@ -697,7 +705,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/decorators.html b/lorax-composer/_modules/pylorax/decorators.html index 8365b735..e7f382cd 100644 --- a/lorax-composer/_modules/pylorax/decorators.html +++ b/lorax-composer/_modules/pylorax/decorators.html @@ -8,7 +8,7 @@ - pylorax.decorators — Lorax 19.7.22 documentation + pylorax.decorators — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -113,7 +113,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/discinfo.html b/lorax-composer/_modules/pylorax/discinfo.html index 6f5df99b..b7d459fd 100644 --- a/lorax-composer/_modules/pylorax/discinfo.html +++ b/lorax-composer/_modules/pylorax/discinfo.html @@ -8,7 +8,7 @@ - pylorax.discinfo — Lorax 19.7.22 documentation + pylorax.discinfo — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -122,7 +122,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/executils.html b/lorax-composer/_modules/pylorax/executils.html index e687bb9f..f4a42ca9 100644 --- a/lorax-composer/_modules/pylorax/executils.html +++ b/lorax-composer/_modules/pylorax/executils.html @@ -8,7 +8,7 @@ - pylorax.executils — Lorax 19.7.22 documentation + pylorax.executils — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -508,7 +508,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/imgutils.html b/lorax-composer/_modules/pylorax/imgutils.html index 42cafb87..d2cbff9d 100644 --- a/lorax-composer/_modules/pylorax/imgutils.html +++ b/lorax-composer/_modules/pylorax/imgutils.html @@ -8,7 +8,7 @@ - pylorax.imgutils — Lorax 19.7.22 documentation + pylorax.imgutils — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -241,7 +241,7 @@ except CalledProcessError: # Problems running losetup are always errors, raise immediately raise - except RuntimeError as e: + except RuntimeError: # Try to setup the loop device 3 times if retries == 3: logger.error("loop_attach failed, retries exhausted.") @@ -569,7 +569,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/installer.html b/lorax-composer/_modules/pylorax/installer.html index 16d8600d..e333053f 100644 --- a/lorax-composer/_modules/pylorax/installer.html +++ b/lorax-composer/_modules/pylorax/installer.html @@ -8,7 +8,7 @@ - pylorax.installer — Lorax 19.7.22 documentation + pylorax.installer — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -70,6 +70,7 @@ log = logging.getLogger("pylorax") import os +import glob import shutil import sys import subprocess @@ -168,7 +169,7 @@ """ def __init__( self, iso, ks_paths, disk_img, img_size=2, kernel_args=None, memory=1024, vnc=None, arch=None, - log_check=None, virtio_host="127.0.0.1", virtio_port=6080, + cancel_func=None, virtio_host="127.0.0.1", virtio_port=6080, qcow2=False, boot_uefi=False, ovmf_path=None): """ Start the installation @@ -263,14 +264,14 @@ # TODO: If vnc has been passed, we should look up the port and print that # for the user at this point - while dom.isActive() and not log_check(): + while dom.isActive() and not (cancel_func and cancel_func()): sys.stdout.write(".") sys.stdout.flush() sleep(10) print - if log_check(): - log.info( "Installation error detected. See logfile." ) + if cancel_func and cancel_func(): + log.info( "Installation error or cancel detected. See logfile." ) else: log.info( "Install finished. Or at least virt shut down." ) @@ -285,19 +286,12 @@ # Undefine the virt, UEFI installs need to have --nvram passed subprocess.call(["virsh", "undefine", self.virt_name, "--nvram"]) +
    -
    [docs]def novirt_install(opts, disk_img, disk_size, repo_url, callback_func=None): +
    [docs]def novirt_install(opts, disk_img, disk_size, repo_url, cancel_func=None): """ Use Anaconda to install to a disk image """ - import selinux - - # Set selinux to Permissive if it is Enforcing - selinux_enforcing = False - if selinux.is_selinux_enabled() and selinux.security_getenforce(): - selinux_enforcing = True - selinux.security_setenforce(0) - # Clean up /tmp/ from previous runs to prevent stale info from being used for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/", "/tmp/yum.root/", "/tmp/yum.pluginconf.d/"]: if os.path.isdir(path): @@ -334,10 +328,14 @@ # Create the sparse image mksparse(disk_img, disk_size * 1024**3) + cancel_funcs = [] + if cancel_func is not None: + cancel_funcs.append(cancel_func) + # Make sure anaconda has the right product and release os.environ["ANACONDA_PRODUCTNAME"] = opts.project os.environ["ANACONDA_PRODUCTVERSION"] = opts.releasever - rc = execWithRedirect("anaconda", args, callback_func=callback_func) + rc = execWithRedirect("anaconda", args, callback_func=lambda : any(f() for f in cancel_funcs)) # Move the anaconda logs over to a log directory log_dir = os.path.abspath(os.path.dirname(opts.logfile)) @@ -358,13 +356,13 @@ if disk_img: dm_name = os.path.splitext(os.path.basename(disk_img))[0] - dm_path = "/dev/mapper/"+dm_name - if os.path.exists(dm_path): - dm_detach(dm_path) - loop_detach(get_loop_name(disk_img)) - if selinux_enforcing: - selinux.security_setenforce(1) + log.debug("Removing device-mapper setup on %s", dm_name) + for d in sorted(glob.glob("/dev/mapper/"+dm_name+"*"), reverse=True): + dm_detach(d) + + log.debug("Removing loop device for %s", disk_img) + loop_detach("/dev/"+get_loop_name(disk_img)) if rc: raise InstallError("novirt_install failed") @@ -394,7 +392,7 @@ execWithRedirect("mv", ["-f", qcow2_img, disk_img], raise_err=True)
    -
    [docs]def virt_install(opts, install_log, disk_img, disk_size): +
    [docs]def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None): """ Use virt-install to install to a disk image @@ -404,6 +402,9 @@ """ iso_mount = IsoMountpoint(opts.iso, opts.location) log_monitor = LogMonitor(install_log) + cancel_funcs = [log_monitor.server.log_check] + if cancel_func is not None: + cancel_funcs.append(cancel_func) kernel_args = "" if opts.kernel_args: @@ -427,7 +428,7 @@ try: virt = VirtualInstall(iso_mount, opts.ks, diskimg_path, disk_size, kernel_args, opts.ram, opts.vnc, opts.arch, - log_check = log_monitor.server.log_check, + cancel_func = lambda : any(f() for f in cancel_funcs), virtio_host = log_monitor.host, virtio_port = log_monitor.port, qcow2=opts.qcow2, boot_uefi=opts.virt_uefi, @@ -443,7 +444,9 @@ iso_mount.umount() if log_monitor.server.log_check(): - raise InstallError("virt_install failed") + raise InstallError("virt_install failed. See logfile.") + elif cancel_func and cancel_func(): + raise InstallError("virt_install canceled by cancel_func") if opts.make_fsimage: mkdiskfsimage(diskimg_path, disk_img, label=opts.fs_label) @@ -493,7 +496,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/logmonitor.html b/lorax-composer/_modules/pylorax/logmonitor.html index 70c719e3..2a0d3724 100644 --- a/lorax-composer/_modules/pylorax/logmonitor.html +++ b/lorax-composer/_modules/pylorax/logmonitor.html @@ -8,7 +8,7 @@ - pylorax.logmonitor — Lorax 19.7.22 documentation + pylorax.logmonitor — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -206,7 +206,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/ltmpl.html b/lorax-composer/_modules/pylorax/ltmpl.html index f7ae5f7c..54edfa6d 100644 --- a/lorax-composer/_modules/pylorax/ltmpl.html +++ b/lorax-composer/_modules/pylorax/ltmpl.html @@ -8,7 +8,7 @@ - pylorax.ltmpl — Lorax 19.7.22 documentation + pylorax.ltmpl — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -121,11 +121,17 @@ # mako template now returns unicode strings lines = map(lambda line: line.encode("utf8"), lines) - # split with shlex and perform brace expansion - lines = map(split_and_expand, lines) - - self.lines = lines - return lines + # split with shlex and perform brace expansion. This can fail, so we unroll the loop + # for better error reporting. + expanded_lines = [] + try: + for line in lines: + expanded_lines.append(split_and_expand(line)) + except Exception as e: + logger.error('shlex error processing "%s": %s', line, str(e)) + raise + self.lines = expanded_lines + return expanded_lines
    [docs]def split_and_expand(line): return [exp for word in shlex.split(line) for exp in brace_expand(word)] @@ -748,7 +754,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/sysutils.html b/lorax-composer/_modules/pylorax/sysutils.html index 951ee555..4938999b 100644 --- a/lorax-composer/_modules/pylorax/sysutils.html +++ b/lorax-composer/_modules/pylorax/sysutils.html @@ -8,7 +8,7 @@ - pylorax.sysutils — Lorax 19.7.22 documentation + pylorax.sysutils — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -191,7 +191,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/treebuilder.html b/lorax-composer/_modules/pylorax/treebuilder.html index 2c79ca95..1682b42c 100644 --- a/lorax-composer/_modules/pylorax/treebuilder.html +++ b/lorax-composer/_modules/pylorax/treebuilder.html @@ -8,7 +8,7 @@ - pylorax.treebuilder — Lorax 19.7.22 documentation + pylorax.treebuilder — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -225,7 +225,8 @@
    [docs]class TreeBuilder(object): '''Builds the arch-specific boot images. inroot should be the installtree root (the newly-built runtime dir)''' - def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None): + def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True, + templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args=""): # NOTE: if you pass an arg named "runtime" to a mako template it'll # clobber some mako internal variables - hence "runtime_img". @@ -234,7 +235,7 @@ inroot=inroot, outroot=outroot, basearch=arch.basearch, libdir=arch.libdir, isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade, - workdir=workdir) + workdir=workdir, extra_boot_args=extra_boot_args) self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir) self._runner.defaults = self.vars self.add_templates = add_templates or [] @@ -403,7 +404,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/treeinfo.html b/lorax-composer/_modules/pylorax/treeinfo.html index f8adf26b..3d6ae729 100644 --- a/lorax-composer/_modules/pylorax/treeinfo.html +++ b/lorax-composer/_modules/pylorax/treeinfo.html @@ -8,7 +8,7 @@ - pylorax.treeinfo — Lorax 19.7.22 documentation + pylorax.treeinfo — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -140,7 +140,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_modules/pylorax/yumhelper.html b/lorax-composer/_modules/pylorax/yumhelper.html index 6ba9e682..14e65c56 100644 --- a/lorax-composer/_modules/pylorax/yumhelper.html +++ b/lorax-composer/_modules/pylorax/yumhelper.html @@ -8,7 +8,7 @@ - pylorax.yumhelper — Lorax 19.7.22 documentation + pylorax.yumhelper — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -38,7 +38,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • @@ -208,7 +208,7 @@
  • modules |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • Module code »
  • pylorax »
  • diff --git a/lorax-composer/_sources/composer-cli.txt b/lorax-composer/_sources/composer-cli.txt index 366e01ad..6c1116fb 100644 --- a/lorax-composer/_sources/composer-cli.txt +++ b/lorax-composer/_sources/composer-cli.txt @@ -40,8 +40,8 @@ compose start http-server qcow2``. It will print a UUID that you can use to keep track of the build. You can also cancel the build if needed. The available types of images is displayed by ``composer-cli compose types``. -Currently this consists of: ext4-filesystem, live-iso, partitioned-disk, qcow2, -tar +Currently this consists of: ami, ext4-filesystem, live-iso, openstack, +partitioned-disk, qcow2, tar, vhd, vmdk Monitor the build status ------------------------ diff --git a/lorax-composer/_sources/lorax-composer.txt b/lorax-composer/_sources/lorax-composer.txt index bc7442b8..bb778583 100644 --- a/lorax-composer/_sources/lorax-composer.txt +++ b/lorax-composer/_sources/lorax-composer.txt @@ -25,6 +25,11 @@ Important Things To Note for information on how to enable it. Otherwise you will see image creation fail to depsolve even if the blueprint itself is correct. +* All image types lock the root account, except for live-iso. You will need to either + use one of the `Customizations`_ methods for setting a ssh key/password, install a + package that creates a user, or use something like `cloud-init` to setup access at + boot time. + Installation ------------ @@ -167,11 +172,25 @@ for selecting optional packages. Customizations ~~~~~~~~~~~~~~ -The ``[[customizations]]`` section can be used to configure the hostname of the final image. eg.:: +The ``[customizations]`` section can be used to configure the hostname of the final image. eg.:: - [[customizations]] + [customizations] hostname = "baseimage" +This is optional and may be left out to use the defaults. + + +[customizations.kernel] +*********************** + +This allows you to append arguments to the bootloader's kernel commandline. This will not have any +effect on ``tar`` or ``ext4-filesystem`` images since they do not include a bootloader. + +For example:: + + [customizations.kernel] + append = "nosmt=force" + [[customizations.sshkey]] ************************* @@ -216,6 +235,96 @@ Add a group to the image. ``name`` is required and ``gid`` is optional:: gid = 1130 +[customizations.timezone] +************************* + +Customizing the timezone and the NTP servers to use for the system:: + + [customizations.timezone] + timezone = "US/Eastern" + ntpservers = ["0.north-america.pool.ntp.org", "1.north-america.pool.ntp.org"] + +The values supported by ``timezone`` can be listed by running ``timedatectl list-timezones``. + +If no timezone is setup the system will default to using `UTC`. The ntp servers are also +optional and will default to using the distribution defaults which are fine for most uses. + +In some image types there are already NTP servers setup, eg. Google cloud image, and they +cannot be overridden because they are required to boot in the selected environment. But the +timezone will be updated to the one selected in the blueprint. + + +[customizations.locale] +*********************** + +Customize the locale settings for the system:: + + [customizations.locale] + languages = ["en_US.UTF-8"] + keyboard = "us" + +The values supported by ``languages`` can be listed by running ``localectl list-locales`` from +the command line. + +The values supported by ``keyboard`` can be listed by running ``localectl list-keymaps`` from +the command line. + +Multiple languages can be added. The first one becomes the +primary, and the others are added as secondary. One or the other of ``languages`` +or ``keyboard`` must be included (or both) in the section. + + +[customizations.firewall] +************************* + +By default the firewall blocks all access except for services that enable their ports explicitly, +like ``sshd``. This command can be used to open other ports or services. Ports are configured using +the port:protocol format:: + + [customizations.firewall] + ports = ["22:tcp", "80:tcp", "imap:tcp", "53:tcp", "53:udp"] + +Numeric ports, or their names from ``/etc/services`` can be used in the ``ports`` enabled/disabled lists. + +The blueprint settings extend any existing settings in the image templates, so if ``sshd`` is +already enabled it will extend the list of ports with the ones listed by the blueprint. + +If the distribution uses ``firewalld`` you can specify services listed by ``firewall-cmd --get-services`` +in a ``customizations.firewall.services`` section:: + + [customizations.firewall.services] + enabled = ["ftp", "ntp", "dhcp"] + disabled = ["telnet"] + +Remember that the ``firewall.services`` are different from the names in ``/etc/services``. + +Both are optional, if they are not used leave them out or set them to an empty list ``[]``. If you +only want the default firewall setup this section can be omitted from the blueprint. + +NOTE: The ``Google`` and ``OpenStack`` templates explicitly disable the firewall for their environment. +This cannot be overridden by the blueprint. + +[customizations.services] +************************* + +This section can be used to control which services are enabled at boot time. +Some image types already have services enabled or disabled in order for the +image to work correctly, and cannot be overridden. eg. ``ami`` requires +``sshd``, ``chronyd``, and ``cloud-init``. Without them the image will not +boot. Blueprint services are added to, not replacing, the list already in the +templates, if any. + + [customizations.services] + enabled = ["sshd", "cockpit", "httpd.service"] + disabled = ["postfix", "telnetd"] + +.. note:: + + The service names are systemd service units. You can only specify the unit + name, eg. ``httpd``, or the full service name, eg. ``httpd.service`` -- but + not other systemd unit files. Note that this is different from newer + releases where you can specify any systemd unit file. + Adding Output Types ------------------- diff --git a/lorax-composer/_sources/pylorax.api.txt b/lorax-composer/_sources/pylorax.api.txt index a2093cdb..5d6db4a0 100644 --- a/lorax-composer/_sources/pylorax.api.txt +++ b/lorax-composer/_sources/pylorax.api.txt @@ -9,6 +9,14 @@ api Package :undoc-members: :show-inheritance: +:mod:`bisect` Module +-------------------- + +.. automodule:: pylorax.api.bisect + :members: + :undoc-members: + :show-inheritance: + :mod:`checkparams` Module ------------------------- diff --git a/lorax-composer/composer-cli.html b/lorax-composer/composer-cli.html index 8840875e..9f56e6af 100644 --- a/lorax-composer/composer-cli.html +++ b/lorax-composer/composer-cli.html @@ -8,7 +8,7 @@ - composer-cli — Lorax 19.7.22 documentation + composer-cli — Lorax 19.7.43 documentation @@ -16,7 +16,7 @@ - + @@ -45,7 +45,7 @@
  • previous |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • @@ -88,8 +88,8 @@ saved by viewing the changelog - compose start http-server qcow2. It will print a UUID that you can use to keep track of the build. You can also cancel the build if needed.

    The available types of images is displayed by composer-cli compose types. -Currently this consists of: ext4-filesystem, live-iso, partitioned-disk, qcow2, -tar

    +Currently this consists of: ami, ext4-filesystem, live-iso, openstack, +partitioned-disk, qcow2, tar, vhd, vmdk

    Monitor the build status¶

    @@ -169,7 +169,7 @@ save the qcow2 image as UUID-disk
  • previous |
  • -
  • Lorax 19.7.22 documentation »
  • +
  • Lorax 19.7.43 documentation »
  • -
    -

    /api/v0/blueprints/info/<blueprint_names>¶

    +
    +

    /api/v0/blueprints/info/<blueprint_names>[?format=<json|toml>]¶

    Return the JSON representation of the blueprint. This includes 3 top level objects. changes which lists whether or not the workspace is different from the most recent commit. blueprints which lists the JSON representation of the blueprint, and errors which will list any errors, like non-existant blueprints.

    -

    Example:

    +

    By default the response is JSON, but if ?format=toml is included in the URL’s +arguments it will return the response as the blueprint’s raw TOML content. +Unless there is an error which will only return a 400 and a standard error +`Status Response`_.

    +

    If there is an error when JSON is requested the successful blueprints and the +errors will both be returned.

    +

    Example of json response:

    {
       "changes": [
         {
    @@ -2409,7 +2894,8 @@ error response with it set to false and an error message included.

    Delete a blueprint. The blueprint is deleted from the branch, and will no longer be listed by the list route. A blueprint can be undeleted using the undo route -to revert to a previous commit.

    +to revert to a previous commit. This will also delete the workspace copy of the +blueprint.

    The response will be a status response with status set to true, or an error response with it set to false and an error message included.

    @@ -3210,7 +3696,21 @@ a line boundry.

    pylorax.api.v0.blueprint_exists(api, branch, blueprint_name)[source]¶
    -
    +

    Return True if the blueprint exists

    + +++ + + + +
    Parameters:
      +
    • api (Flask) – flask object
    • +
    • branch (str) – Branch name
    • +
    • recipe_name (str) – Recipe name to read
    • +
    +
    +
    @@ -3349,6 +3849,34 @@ a line boundry.

    yumbase Module¶

    +
    +
    +class pylorax.api.yumbase.YumLock(conf, expire_secs=21600)[source]¶
    +

    Bases: object

    +

    Hold the YumBase object and a Lock to control access to it.

    +

    self.yb is a property that returns the YumBase object, but it may change +from one call to the next if the upstream repositories have changed.

    +
    +
    +lock[source]¶
    +

    Check for repo updates (using expiration time) and return the lock

    +

    If the repository has been updated, tear down the old YumBase and +create a new one. This is the only way to force yum to use the new +metadata.

    +
    + +
    +
    +lock_check[source]¶
    +

    Force a check for repo updates and return the lock

    +

    If the repository has been updated, tear down the old YumBase and +create a new one. This is the only way to force yum to use the new +metadata.

    +

    Use this method sparingly, it removes the repodata and downloads a new copy every time.

    +
    + +
    +
    pylorax.api.yumbase.get_base_object(conf)[source]¶
    @@ -3398,6 +3926,7 @@ a line boundry.

    @@ -107,6 +107,12 @@ pylorax.get_buildarch(ybo)[source]¶
    +
    +
    +pylorax.log_selinux_state()[source]¶
    +

    Log the current state of selinux

    +
    +

    base Module¶

    @@ -116,27 +122,27 @@

    Bases: object

    -pcritical(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f838cf59150>)[source]¶
    +pcritical(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f44874d0150>)[source]¶
    -pdebug(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f838cf59150>)[source]¶
    +pdebug(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f44874d0150>)[source]¶
    -perror(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f838cf59150>)[source]¶
    +perror(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f44874d0150>)[source]¶
    -pinfo(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f838cf59150>)[source]¶
    +pinfo(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f44874d0150>)[source]¶
    -pwarning(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f838cf59150>)[source]¶
    +pwarning(msg, fobj=<open file '<stdout>', mode 'w' at 0x7f44874d0150>)[source]¶
    @@ -275,7 +281,7 @@ releasever Release version, passed to template. Default is 17

    -pylorax.creator.make_image(opts, ks, callback_func=None)[source]¶
    +pylorax.creator.make_image(opts, ks, cancel_func=None)[source]¶

    Install to an image

    Use virt or anaconda to install to an image.

    Returns the full path of of the image created.

    @@ -370,7 +376,7 @@ type img_mount: imgutils.PartitionMount

    -pylorax.creator.run_creator(opts, callback_func=None)[source]¶
    +pylorax.creator.run_creator(opts, cancel_func=None)[source]¶

    Run the image creator process

    @@ -775,7 +781,7 @@ initrd.img than the iso has. The iso is still used for stage2.

    -class pylorax.installer.VirtualInstall(iso, ks_paths, disk_img, img_size=2, kernel_args=None, memory=1024, vnc=None, arch=None, log_check=None, virtio_host='127.0.0.1', virtio_port=6080, qcow2=False, boot_uefi=False, ovmf_path=None)[source]¶
    +class pylorax.installer.VirtualInstall(iso, ks_paths, disk_img, img_size=2, kernel_args=None, memory=1024, vnc=None, arch=None, cancel_func=None, virtio_host='127.0.0.1', virtio_port=6080, qcow2=False, boot_uefi=False, ovmf_path=None)[source]¶

    Bases: object

    Run virt-install using an iso and kickstart(s)

    @@ -789,13 +795,13 @@ initrd.img than the iso has. The iso is still used for stage2.

    -pylorax.installer.novirt_install(opts, disk_img, disk_size, repo_url, callback_func=None)[source]¶
    +pylorax.installer.novirt_install(opts, disk_img, disk_size, repo_url, cancel_func=None)[source]¶

    Use Anaconda to install to a disk image

    -pylorax.installer.virt_install(opts, install_log, disk_img, disk_size)[source]¶
    +pylorax.installer.virt_install(opts, install_log, disk_img, disk_size, cancel_func=None)[source]¶

    Use virt-install to install to a disk image

    install_log is the path to write the log from virt-install disk_img is the full path to the final disk or filesystem image @@ -1320,7 +1326,7 @@ Example:

    -class pylorax.treebuilder.TreeBuilder(product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None)[source]¶
    +class pylorax.treebuilder.TreeBuilder(product, arch, inroot, outroot, runtime, isolabel, domacboot=False, doupgrade=True, templatedir=None, add_templates=None, add_template_vars=None, workdir=None, extra_boot_args='')[source]¶

    Bases: object

    Builds the arch-specific boot images. inroot should be the installtree root (the newly-built runtime dir)

    @@ -1467,6 +1473,7 @@ image is built with the filename “${prefix}-${kernel.version}.img”
  • api Package