diff --git a/docs/html/.buildinfo b/docs/html/.buildinfo index dbe3bfab..dc28eac6 100644 --- a/docs/html/.buildinfo +++ b/docs/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: f75ad3ae64910413db55774b9b88758b +config: 1bf42a3192ec6a62d96115c18e727069 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/html/.doctrees/composer-cli.doctree b/docs/html/.doctrees/composer-cli.doctree index 2a2a2ab3..aff67704 100644 Binary files a/docs/html/.doctrees/composer-cli.doctree and b/docs/html/.doctrees/composer-cli.doctree differ diff --git a/docs/html/.doctrees/composer.cli.doctree b/docs/html/.doctrees/composer.cli.doctree index 4c8743cc..20f86cfe 100644 Binary files a/docs/html/.doctrees/composer.cli.doctree and b/docs/html/.doctrees/composer.cli.doctree differ diff --git a/docs/html/.doctrees/composer.doctree b/docs/html/.doctrees/composer.doctree index cb76565a..37ef746c 100644 Binary files a/docs/html/.doctrees/composer.doctree and b/docs/html/.doctrees/composer.doctree differ diff --git a/docs/html/.doctrees/environment.pickle b/docs/html/.doctrees/environment.pickle index 504e8c7a..e1d9a217 100644 Binary files a/docs/html/.doctrees/environment.pickle and b/docs/html/.doctrees/environment.pickle differ diff --git a/docs/html/.doctrees/index.doctree b/docs/html/.doctrees/index.doctree index 661acbad..ccf4e0a1 100644 Binary files a/docs/html/.doctrees/index.doctree and b/docs/html/.doctrees/index.doctree differ diff --git a/docs/html/.doctrees/intro.doctree b/docs/html/.doctrees/intro.doctree index 3fed1889..4bb3564e 100644 Binary files a/docs/html/.doctrees/intro.doctree and b/docs/html/.doctrees/intro.doctree differ diff --git a/docs/html/.doctrees/livemedia-creator.doctree b/docs/html/.doctrees/livemedia-creator.doctree index 8bb5274e..15612b08 100644 Binary files a/docs/html/.doctrees/livemedia-creator.doctree and b/docs/html/.doctrees/livemedia-creator.doctree differ diff --git a/docs/html/.doctrees/lorax-composer.doctree b/docs/html/.doctrees/lorax-composer.doctree index aeb7694c..7e277f5a 100644 Binary files a/docs/html/.doctrees/lorax-composer.doctree and b/docs/html/.doctrees/lorax-composer.doctree differ diff --git a/docs/html/.doctrees/lorax.doctree b/docs/html/.doctrees/lorax.doctree index 76b9fc56..a66cab87 100644 Binary files a/docs/html/.doctrees/lorax.doctree and b/docs/html/.doctrees/lorax.doctree differ diff --git a/docs/html/.doctrees/modules.doctree b/docs/html/.doctrees/modules.doctree index 5894f5a4..51a4d622 100644 Binary files a/docs/html/.doctrees/modules.doctree and b/docs/html/.doctrees/modules.doctree differ diff --git a/docs/html/.doctrees/product-images.doctree b/docs/html/.doctrees/product-images.doctree index 76279ee0..91f72de6 100644 Binary files a/docs/html/.doctrees/product-images.doctree and b/docs/html/.doctrees/product-images.doctree differ diff --git a/docs/html/.doctrees/pylorax.api.doctree b/docs/html/.doctrees/pylorax.api.doctree index 2bccacb2..3eec3d23 100644 Binary files a/docs/html/.doctrees/pylorax.api.doctree and b/docs/html/.doctrees/pylorax.api.doctree differ diff --git a/docs/html/.doctrees/pylorax.doctree b/docs/html/.doctrees/pylorax.doctree index 49d8ac20..fcbca090 100644 Binary files a/docs/html/.doctrees/pylorax.doctree and b/docs/html/.doctrees/pylorax.doctree differ diff --git a/docs/html/_modules/composer/cli.html b/docs/html/_modules/composer/cli.html index 670e3d34..19343efd 100644 --- a/docs/html/_modules/composer/cli.html +++ b/docs/html/_modules/composer/cli.html @@ -8,7 +8,7 @@ - composer.cli — Lorax 29.21 documentation + composer.cli — Lorax 29.28 documentation @@ -56,7 +56,7 @@
- 29.21 + 29.28
@@ -244,7 +244,7 @@ + + + + + + +
+ + + + +
+ + + + + +
+ +
+ + + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +

Source code for pylorax.api.bisect

+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
[docs]def insort_left(a, x, key=None, lo=0, hi=None): + """Insert item x in list a, and keep it sorted assuming a is sorted. + + :param a: sorted list + :type a: list + :param x: item to insert into the list + :type x: object + :param key: Function to use to compare items in the list + :type key: function + :returns: index where the item was inserted + :rtype: int + + If x is already in a, insert it to the left of the leftmost x. + Optional args lo (default 0) and hi (default len(a)) bound the + slice of a to be searched. + + This is a modified version of bisect.insort_left that can use a + function for the compare, and returns the index position where it + was inserted. + """ + if key is None: + key = lambda i: i + + if lo < 0: + raise ValueError('lo must be non-negative') + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if key(a[mid]) < key(x): lo = mid+1 + else: hi = mid + a.insert(lo, x) + return lo
+
+ +
+ +
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/api/checkparams.html b/docs/html/_modules/pylorax/api/checkparams.html index c1b99759..dd17a986 100644 --- a/docs/html/_modules/pylorax/api/checkparams.html +++ b/docs/html/_modules/pylorax/api/checkparams.html @@ -8,7 +8,7 @@ - pylorax.api.checkparams — Lorax 29.21 documentation + pylorax.api.checkparams — Lorax 29.28 documentation @@ -56,7 +56,7 @@
- 29.21 + 29.28
@@ -234,7 +234,7 @@ + + + + + + +
+ + + + +
+ + + + + +
+ +
+ + + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +

Source code for pylorax.api.gitrpm

+# Copyright (C) 2019 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+""" Clone a git repository and package it as an rpm
+
+This module contains functions for cloning a git repo, creating a tar archive of
+the selected commit, branch, or tag, and packaging the files into an rpm that will
+be installed by anaconda when creating the image.
+"""
+import logging
+log = logging.getLogger("lorax-composer")
+
+import os
+from rpmfluff import SimpleRpmBuild
+import shutil
+import subprocess
+import tempfile
+import time
+
+from pylorax.sysutils import joinpaths
+
+
[docs]def get_repo_description(gitRepo): + """ Return a description including the git repo and reference + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + :returns: A string with the git repo url and reference + :rtype: str + """ + return "Created from %s, reference '%s', on %s" % (gitRepo["repo"], gitRepo["ref"], time.ctime())
+ +
[docs]class GitArchiveTarball: + """Create a git archive of the selected git repo and reference""" + def __init__(self, gitRepo): + self._gitRepo = gitRepo + self.sourceName = self._gitRepo["rpmname"]+".tar.xz" + +
[docs] def write_file(self, sourcesDir): + """ Create the tar archive + + :param sourcesDir: Path to use for creating the archive + :type sourcesDir: str + + This clones the git repository and creates a git archive from the specified reference. + The result is in RPMNAME.tar.xz under the sourcesDir + """ + # Clone the repository into a temporary location + cmd = ["git", "clone", self._gitRepo["repo"], joinpaths(sourcesDir, "gitrepo")] + log.debug(cmd) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + log.error("Failed to clone %s: %s", self._gitRepo["repo"], e.output) + raise RuntimeError("Failed to clone %s" % self._gitRepo["repo"]) + + oldcwd = os.getcwd() + try: + os.chdir(joinpaths(sourcesDir, "gitrepo")) + + # Configure archive to create a .tar.xz + cmd = ["git", "config", "tar.tar.xz.command", "xz -c"] + log.debug(cmd) + subprocess.check_call(cmd) + + cmd = ["git", "archive", "--prefix", self._gitRepo["rpmname"] + "/", "-o", joinpaths(sourcesDir, self.sourceName), self._gitRepo["ref"]] + log.debug(cmd) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + log.error("Failed to archive %s: %s", self._gitRepo["repo"], e.output) + raise RuntimeError("Failed to clone %s" % self._gitRepo["repo"]) + finally: + # Cleanup even if there was an error + os.chdir(oldcwd) + shutil.rmtree(joinpaths(sourcesDir, "gitrepo"))
+ +
[docs]class GitRpmBuild(SimpleRpmBuild): + """Build an rpm containing files from a git repository""" + def __init__(self, *args, **kwargs): + self._base_dir = None + super().__init__(*args, **kwargs) + +
[docs] def check(self): + raise NotImplementedError
+ +
[docs] def get_base_dir(self): + """Place all the files under a temporary directory + rpmbuild/ + """ + if not self._base_dir: + self._base_dir = tempfile.mkdtemp(prefix="lorax-git-rpm.") + return joinpaths(self._base_dir, "rpmbuild")
+ +
[docs] def cleanup_tmpdir(self): + """Remove the temporary directory and all of its contents + """ + if len(self._base_dir) < 5: + raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir()) + + shutil.rmtree(self._base_dir)
+ +
[docs] def clean(self): + """Remove the base directory from inside the tmpdir""" + if len(self.get_base_dir()) < 5: + raise RuntimeError("Invalid base_dir: %s" % self.get_base_dir()) + shutil.rmtree(self.get_base_dir(), ignore_errors=True)
+ +
[docs] def add_git_tarball(self, gitRepo): + """Add a tar archive of a git repository to the rpm + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + + This populates the rpm with the URL of the git repository, the summary + describing the repo, the description of the repository and reference used, + and sets up the rpm to install the archive contents into the destination + path. + """ + self.addUrl(gitRepo["repo"]) + self.add_summary(gitRepo["summary"]) + self.add_description(get_repo_description(gitRepo)) + self.addLicense("Unknown") + sourceIndex = self.add_source(GitArchiveTarball(gitRepo)) + self.section_build += "tar -xvf %s\n" % self.sources[sourceIndex].sourceName + dest = os.path.normpath(gitRepo["destination"]) + # Prevent double slash root + if dest == "/": + dest = "" + self.create_parent_dirs(dest) + self.section_install += "cp -r %s/. $RPM_BUILD_ROOT/%s\n" % (gitRepo["rpmname"], dest) + sub = self.get_subpackage(None) + if not dest: + # / is special, we don't want to include / itself, just what's under it + sub.section_files += "/*\n" + else: + sub.section_files += "%s/\n" % dest
+ +
[docs]def make_git_rpm(gitRepo, dest): + """ Create an rpm from the specified git repo + + :param gitRepo: A dict with the repository details + :type gitRepo: dict + + This will clone the git repository, create an archive of the selected reference, + and build an rpm that will install the files from the repository under the destination + directory. The gitRepo dict should have the following fields:: + + rpmname: "server-config" + rpmversion: "1.0" + rpmrelease: "1" + summary: "Setup files for server deployment" + repo: "PATH OF GIT REPO TO CLONE" + ref: "v1.0" + destination: "/opt/server/" + + * rpmname: Name of the rpm to create, also used as the prefix name in the tar archive + * rpmversion: Version of the rpm, eg. "1.0.0" + * rpmrelease: Release of the rpm, eg. "1" + * summary: Summary string for the rpm + * repo: URL of the get repo to clone and create the archive from + * ref: Git reference to check out. eg. origin/branch-name, git tag, or git commit hash + * destination: Path to install the / of the git repo at when installing the rpm + """ + gitRpm = GitRpmBuild(gitRepo["rpmname"], gitRepo["rpmversion"], gitRepo["rpmrelease"], ["noarch"]) + try: + gitRpm.add_git_tarball(gitRepo) + gitRpm.do_make() + rpmfile = gitRpm.get_built_rpm("noarch") + shutil.move(rpmfile, dest) + except Exception as e: + log.error("Creating git repo rpm: %s", e) + raise RuntimeError("Creating git repo rpm: %s" % e) + finally: + gitRpm.cleanup_tmpdir() + + return os.path.basename(rpmfile)
+ +# Create the git rpms, if any, and return the path to the repo under results_dir +
[docs]def create_gitrpm_repo(results_dir, recipe): + """Create a dnf repository with the rpms from the recipe + + :param results_dir: Path to create the repository under + :type results_dir: str + :param recipe: The recipe to get the repos.git entries from + :type recipe: Recipe + :returns: Path to the dnf repository or "" + :rtype: str + + This function creates a dnf repository directory at results_dir+"repo/", + creates rpms for all of the repos.git entries in the recipe, runs createrepo_c + on the dnf repository so that Anaconda can use it, and returns the path to the + repository to the caller. + """ + if "repos" not in recipe or "git" not in recipe["repos"]: + return "" + + gitrepo = joinpaths(results_dir, "repo/") + if not os.path.exists(gitrepo): + os.makedirs(gitrepo) + for r in recipe["repos"]["git"]: + make_git_rpm(r, gitrepo) + cmd = ["createrepo_c", gitrepo] + log.debug(cmd) + try: + subprocess.check_output(cmd) + except subprocess.CalledProcessError as e: + log.error("Failed to create repo at %s: %s", gitrepo, e.output) + raise RuntimeError("Failed to create repo at %s" % gitrepo) + + return gitrepo
+
+ +
+ +
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/html/_modules/pylorax/api/projects.html b/docs/html/_modules/pylorax/api/projects.html index 83b77368..0d538c67 100644 --- a/docs/html/_modules/pylorax/api/projects.html +++ b/docs/html/_modules/pylorax/api/projects.html @@ -8,7 +8,7 @@ - pylorax.api.projects — Lorax 29.21 documentation + pylorax.api.projects — Lorax 29.28 documentation @@ -56,7 +56,7 @@
- 29.21 + 29.28
@@ -173,12 +173,14 @@ import logging log = logging.getLogger("lorax-composer") -import os from configparser import ConfigParser import dnf from glob import glob +import os import time +from pylorax.api.bisect import insort_left + TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" @@ -231,6 +233,32 @@ "upstream_vcs": "UPSTREAM_VCS"} +
[docs]def pkg_to_build(pkg): + """Extract the build details from a hawkey.Package object + + :param pkg: hawkey.Package object with package details + :type pkg: hawkey.Package + :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ... + :rtype: dict + + metadata entries are hard-coded to {} + + Note that this only returns the build dict, it does not include the name, description, etc. + """ + return {"epoch": pkg.epoch, + "release": pkg.release, + "arch": pkg.arch, + "build_time": api_time(pkg.buildtime), + "changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package + "build_config_ref": "BUILD_CONFIG_REF", + "build_env_ref": "BUILD_ENV_REF", + "metadata": {}, + "source": {"license": pkg.license, + "version": pkg.version, + "source_ref": "SOURCE_REF", + "metadata": {}}}
+ +
[docs]def pkg_to_project_info(pkg): """Extract the details from a hawkey.Package object @@ -241,25 +269,12 @@ metadata entries are hard-coded to {} """ - build = {"epoch": pkg.epoch, - "release": pkg.release, - "arch": pkg.arch, - "build_time": api_time(pkg.buildtime), - "changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package - "build_config_ref": "BUILD_CONFIG_REF", - "build_env_ref": "BUILD_ENV_REF", - "metadata": {}, - "source": {"license": pkg.license, - "version": pkg.version, - "source_ref": "SOURCE_REF", - "metadata": {}}} - return {"name": pkg.name, "summary": pkg.summary, "description": pkg.description, "homepage": pkg.url, "upstream_vcs": "UPSTREAM_VCS", - "builds": [build]}
+ "builds": [pkg_to_build(pkg)]}
[docs]def pkg_to_dep(pkg): @@ -336,7 +351,24 @@ pkgs = dbo.sack.query().available().filter(name__glob=project_names) else: pkgs = dbo.sack.query().available() - return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
+ + # iterate over pkgs + # - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position + # - if pkg.name is already in results, get its builds. If the build for pkg is different + # in any way (version, arch, etc.) add it to the entry's builds list. If it is the same, + # skip it. + results = [] + results_names = {} + for p in pkgs: + if p.name.lower() not in results_names: + idx = insort_left(results, pkg_to_project_info(p), key=lambda p: p["name"].lower()) + results_names[p.name.lower()] = idx + else: + build = pkg_to_build(p) + if build not in results[results_names[p.name.lower()]]["builds"]: + results[results_names[p.name.lower()]]["builds"].append(build) + + return results def _depsolve(dbo, projects, groups): """Add projects to a new transaction @@ -477,9 +509,7 @@ """ # TODO - Figure out what to do with this for Fedora 'modules' - projs = projects_info(dbo, module_names) - return sorted(map(proj_to_module, projs), key=lambda p: p["name"].lower()) - + return list(map(proj_to_module, projects_info(dbo, module_names)))
[docs]def modules_info(dbo, module_names): """Return details about a module, including dependencies @@ -743,7 +773,7 @@ @@ -58,7 +58,7 @@
- 29.21 + 29.28
@@ -166,14 +166,14 @@ of the iso for all of the supported architectures.

Before Lorax

-

Tree building tools such as pungi and revisor rely on ‘buildinstall’ in +

Tree building tools such as pungi and revisor rely on 'buildinstall' in anaconda/scripts/ to produce the boot images and other such control files in the final tree. The existing buildinstall scripts written in a mix of bash and Python are unmaintainable. Lorax is an attempt to replace them with something more flexible.

EXISTING WORKFLOW:

pungi and other tools call scripts/buildinstall, which in turn call other -scripts to do the image building and data generation. Here’s how it +scripts to do the image building and data generation. Here's how it currently looks:

@@ -200,7 +200,7 @@ maketreeinfo.py, makestamp.py, and buildinstall First, almost all knowledge of what goes in to the stage 1 and stage 2 images lives in upd-instroot. The mk-images* scripts copy things from the root created by upd-instroot in order to build the stage 1 image, though -it’s not completely clear from reading the scripts.

+it's not completely clear from reading the scripts.

NEW IDEAS:

Create a new central driver with all information living in Python modules. Configuration files will provide the knowledge previously contained in the @@ -218,7 +218,7 @@ upd-instroot and mk-images* scripts.

- +
@@ -251,7 +251,7 @@ upd-instroot and mk-images* scripts.