From 0e54983be8483a800c2bbc3370059998849fc03e Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Thu, 6 Dec 2018 11:13:36 -0800 Subject: [PATCH] lorax-composer: Handle packages with multiple builds When the repository has multiple arches, eg. i686 and x86_64, it should add a new entry to the project's builds list, not create a new project in the list. This handles that by adding a modified insort_left function and examining the packages returned from dnf to make sure they aren't already listed in the results. It also handles adding them in sorted order so that no further sorting needs to be done on the results. Resolves: rhbz#1657055 (cherry picked from commit 663a0dcd73494b8eaea9be12750c63b07553b524) --- src/pylorax/api/bisect.py | 49 +++++++++++++++++++ src/pylorax/api/projects.py | 94 +++++++++++++++++++++++++++--------- tests/pylorax/test_bisect.py | 41 ++++++++++++++++ 3 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 src/pylorax/api/bisect.py create mode 100644 tests/pylorax/test_bisect.py diff --git a/src/pylorax/api/bisect.py b/src/pylorax/api/bisect.py new file mode 100644 index 00000000..a1bbdcc0 --- /dev/null +++ b/src/pylorax/api/bisect.py @@ -0,0 +1,49 @@ +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +def insort_left(a, x, key=None, lo=0, hi=None): + """Insert item x in list a, and keep it sorted assuming a is sorted. + + :param a: sorted list + :type a: list + :param x: item to insert into the list + :type x: object + :param key: Function to use to compare items in the list + :type key: function + :returns: index where the item was inserted + :rtype: int + + If x is already in a, insert it to the left of the leftmost x. + Optional args lo (default 0) and hi (default len(a)) bound the + slice of a to be searched. + + This is a modified version of bisect.insort_left that can use a + function for the compare, and returns the index position where it + was inserted. + """ + if key is None: + key = lambda i: i + + if lo < 0: + raise ValueError('lo must be non-negative') + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if key(a[mid]) < key(x): lo = mid+1 + else: hi = mid + a.insert(lo, x) + return lo diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 0fa1a75c..6b30c27d 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -17,13 +17,14 @@ import logging log = logging.getLogger("lorax-composer") -import os from ConfigParser import ConfigParser import fnmatch from glob import glob +import os import time from yum.Errors import YumBaseError +from pylorax.api.bisect import insort_left TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" @@ -77,6 +78,32 @@ def yaps_to_project(yaps): "upstream_vcs": "UPSTREAM_VCS"} +def yaps_to_build(yaps): + """Extract the build details from a hawkey.Package object + + :param yaps: Yum object with package details + :type yaps: YumAvailablePackageSqlite + :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ... + :rtype: dict + + metadata entries are hard-coded to {} + + Note that this only returns the build dict, it does not include the name, description, etc. + """ + return {"epoch": int(yaps.epoch), + "release": yaps.release, + "arch": yaps.arch, + "build_time": api_time(yaps.buildtime), + "changelog": api_changelog(yaps.returnChangelog()), + "build_config_ref": "BUILD_CONFIG_REF", + "build_env_ref": "BUILD_ENV_REF", + "metadata": {}, + "source": {"license": yaps.license, + "version": yaps.version, + "source_ref": "SOURCE_REF", + "metadata": {}}} + + def yaps_to_project_info(yaps): """Extract the details from a YumAvailablePackageSqlite object @@ -87,25 +114,12 @@ def yaps_to_project_info(yaps): metadata entries are hard-coded to {} """ - build = {"epoch": int(yaps.epoch), - "release": yaps.release, - "arch": yaps.arch, - "build_time": api_time(yaps.buildtime), - "changelog": api_changelog(yaps.returnChangelog()), - "build_config_ref": "BUILD_CONFIG_REF", - "build_env_ref": "BUILD_ENV_REF", - "metadata": {}, - "source": {"license": yaps.license, - "version": yaps.version, - "source_ref": "SOURCE_REF", - "metadata": {}}} - return {"name": yaps.name, "summary": yaps.summary, "description": yaps.description, "homepage": yaps.url, "upstream_vcs": "UPSTREAM_VCS", - "builds": [build]} + "builds": [yaps_to_build(yaps)]} def tm_to_dep(tm): @@ -170,7 +184,6 @@ def projects_list(yb): yb.closeRpmDB() return sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower()) - def projects_info(yb, project_names): """Return details about specific projects @@ -187,7 +200,25 @@ def projects_info(yb, project_names): raise ProjectsError("There was a problem with info for %s: %s" % (project_names, str(e))) finally: yb.closeRpmDB() - return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower()) + + # iterate over pkgs + # - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position + # - if pkg.name is already in results, get its builds. If the build for pkg is different + # in any way (version, arch, etc.) add it to the entry's builds list. If it is the same, + # skip it. + results = [] + results_names = {} + for p in ybl.available: + if p.name.lower() not in results_names: + idx = insort_left(results, yaps_to_project_info(p), key=lambda p: p["name"].lower()) + results_names[p.name.lower()] = idx + else: + build = yaps_to_build(p) + if build not in results[results_names[p.name.lower()]]["builds"]: + results[results_names[p.name.lower()]]["builds"].append(build) + + return results + def filterVersionGlob(pkgs, version): """Filter a list of yum package objects with a version glob @@ -373,14 +404,29 @@ def modules_list(yb, module_names): Modules don't exist in RHEL7 so this only returns projects and sets the type to "rpm" """ - try: - ybl = yb.doPackageLists(pkgnarrow="available", patterns=module_names, showdups=False) - except YumBaseError as e: - raise ProjectsError("There was a problem listing modules: %s" % str(e)) - finally: - yb.closeRpmDB() - return sorted(map(yaps_to_module, ybl.available), key=lambda p: p["name"].lower()) + projs = _unique_dicts(projects_info(yb, module_names), key=lambda p: p["name"].lower()) + return list(map(yaps_to_module, projs)) +def _unique_dicts(lst, key): + """Return a new list of dicts, only including one match of key(d) + + :param lst: list of dicts + :type lst: list + :param key: key function to match lst entries + :type key: function + :returns: list of the unique lst entries + :rtype: list + + Uses key(d) to test for duplicates in the returned list, creating a + list of unique return values. + """ + result = [] + result_keys = [] + for d in lst: + if key(d) not in result_keys: + result.append(d) + result_keys.append(key(d)) + return result def modules_info(yb, module_names): """Return details about a module, including dependencies diff --git a/tests/pylorax/test_bisect.py b/tests/pylorax/test_bisect.py new file mode 100644 index 00000000..8e5a101a --- /dev/null +++ b/tests/pylorax/test_bisect.py @@ -0,0 +1,41 @@ +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import unittest + +from pylorax.api.bisect import insort_left + + +class BisectTest(unittest.TestCase): + def test_insort_left_nokey(self): + results = [] + for x in range(0, 10): + insort_left(results, x) + self.assertEqual(results, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + def test_insort_left_key_strings(self): + unsorted = ["Maggie", "Homer", "Bart", "Marge"] + results = [] + for x in unsorted: + insort_left(results, x, key=lambda p: p.lower()) + self.assertEqual(results, ["Bart", "Homer", "Maggie", "Marge"]) + + def test_insort_left_key_dict(self): + unsorted = [{"name":"Maggie"}, {"name":"Homer"}, {"name":"Bart"}, {"name":"Marge"}] + results = [] + for x in unsorted: + insort_left(results, x, key=lambda p: p["name"].lower()) + self.assertEqual(results, [{"name":"Bart"}, {"name":"Homer"}, {"name":"Maggie"}, {"name":"Marge"}])