From ed611933cb70191b51db64e8ef63019efbc04b18 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Thu, 6 Dec 2018 11:13:36 -0800 Subject: [PATCH] lorax-composer: Handle packages with multiple builds When the repository has multiple arches, eg. i686 and x86_64, it should add a new entry to the project's builds list, not create a new project in the list. This handles that by adding a modified insort_left function and examining the packages returned from dnf to make sure they aren't already listed in the results. It also handles adding them in sorted order so that no further sorting needs to be done on the results. Resolves: rhbz#1656642 (cherry picked from commit d18934775c100fd96d5dd66292a15739cd0bf69a) --- src/pylorax/api/bisect.py | 49 ++++++++++++++++++++ src/pylorax/api/projects.py | 88 ++++++++++++++++++++++++++++-------- tests/pylorax/test_bisect.py | 41 +++++++++++++++++ 3 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 src/pylorax/api/bisect.py create mode 100644 tests/pylorax/test_bisect.py diff --git a/src/pylorax/api/bisect.py b/src/pylorax/api/bisect.py new file mode 100644 index 00000000..a1bbdcc0 --- /dev/null +++ b/src/pylorax/api/bisect.py @@ -0,0 +1,49 @@ +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +def insort_left(a, x, key=None, lo=0, hi=None): + """Insert item x in list a, and keep it sorted assuming a is sorted. + + :param a: sorted list + :type a: list + :param x: item to insert into the list + :type x: object + :param key: Function to use to compare items in the list + :type key: function + :returns: index where the item was inserted + :rtype: int + + If x is already in a, insert it to the left of the leftmost x. + Optional args lo (default 0) and hi (default len(a)) bound the + slice of a to be searched. + + This is a modified version of bisect.insort_left that can use a + function for the compare, and returns the index position where it + was inserted. + """ + if key is None: + key = lambda i: i + + if lo < 0: + raise ValueError('lo must be non-negative') + if hi is None: + hi = len(a) + while lo < hi: + mid = (lo+hi)//2 + if key(a[mid]) < key(x): lo = mid+1 + else: hi = mid + a.insert(lo, x) + return lo diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py index 2072be41..22b1320e 100644 --- a/src/pylorax/api/projects.py +++ b/src/pylorax/api/projects.py @@ -17,12 +17,14 @@ import logging log = logging.getLogger("lorax-composer") -import os from configparser import ConfigParser import dnf from glob import glob +import os import time +from pylorax.api.bisect import insort_left + TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" @@ -75,6 +77,32 @@ def pkg_to_project(pkg): "upstream_vcs": "UPSTREAM_VCS"} +def pkg_to_build(pkg): + """Extract the build details from a hawkey.Package object + + :param pkg: hawkey.Package object with package details + :type pkg: hawkey.Package + :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ... + :rtype: dict + + metadata entries are hard-coded to {} + + Note that this only returns the build dict, it does not include the name, description, etc. + """ + return {"epoch": pkg.epoch, + "release": pkg.release, + "arch": pkg.arch, + "build_time": api_time(pkg.buildtime), + "changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package + "build_config_ref": "BUILD_CONFIG_REF", + "build_env_ref": "BUILD_ENV_REF", + "metadata": {}, + "source": {"license": pkg.license, + "version": pkg.version, + "source_ref": "SOURCE_REF", + "metadata": {}}} + + def pkg_to_project_info(pkg): """Extract the details from a hawkey.Package object @@ -85,25 +113,12 @@ def pkg_to_project_info(pkg): metadata entries are hard-coded to {} """ - build = {"epoch": pkg.epoch, - "release": pkg.release, - "arch": pkg.arch, - "build_time": api_time(pkg.buildtime), - "changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package - "build_config_ref": "BUILD_CONFIG_REF", - "build_env_ref": "BUILD_ENV_REF", - "metadata": {}, - "source": {"license": pkg.license, - "version": pkg.version, - "source_ref": "SOURCE_REF", - "metadata": {}}} - return {"name": pkg.name, "summary": pkg.summary, "description": pkg.description, "homepage": pkg.url, "upstream_vcs": "UPSTREAM_VCS", - "builds": [build]} + "builds": [pkg_to_build(pkg)]} def pkg_to_dep(pkg): @@ -180,7 +195,24 @@ def projects_info(dbo, project_names): pkgs = dbo.sack.query().available().filter(name__glob=project_names) else: pkgs = dbo.sack.query().available() - return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower()) + + # iterate over pkgs + # - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position + # - if pkg.name is already in results, get its builds. If the build for pkg is different + # in any way (version, arch, etc.) add it to the entry's builds list. If it is the same, + # skip it. + results = [] + results_names = {} + for p in pkgs: + if p.name.lower() not in results_names: + idx = insort_left(results, pkg_to_project_info(p), key=lambda p: p["name"].lower()) + results_names[p.name.lower()] = idx + else: + build = pkg_to_build(p) + if build not in results[results_names[p.name.lower()]]["builds"]: + results[results_names[p.name.lower()]]["builds"].append(build) + + return results def _depsolve(dbo, projects, groups): """Add projects to a new transaction @@ -319,9 +351,29 @@ def modules_list(dbo, module_names): """ # TODO - Figure out what to do with this for Fedora 'modules' - projs = projects_info(dbo, module_names) - return sorted(map(proj_to_module, projs), key=lambda p: p["name"].lower()) + projs = _unique_dicts(projects_info(dbo, module_names), key=lambda p: p["name"].lower()) + return list(map(proj_to_module, projs)) +def _unique_dicts(lst, key): + """Return a new list of dicts, only including one match of key(d) + + :param lst: list of dicts + :type lst: list + :param key: key function to match lst entries + :type key: function + :returns: list of the unique lst entries + :rtype: list + + Uses key(d) to test for duplicates in the returned list, creating a + list of unique return values. + """ + result = [] + result_keys = [] + for d in lst: + if key(d) not in result_keys: + result.append(d) + result_keys.append(key(d)) + return result def modules_info(dbo, module_names): """Return details about a module, including dependencies diff --git a/tests/pylorax/test_bisect.py b/tests/pylorax/test_bisect.py new file mode 100644 index 00000000..8e5a101a --- /dev/null +++ b/tests/pylorax/test_bisect.py @@ -0,0 +1,41 @@ +# +# Copyright (C) 2018 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import unittest + +from pylorax.api.bisect import insort_left + + +class BisectTest(unittest.TestCase): + def test_insort_left_nokey(self): + results = [] + for x in range(0, 10): + insort_left(results, x) + self.assertEqual(results, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + + def test_insort_left_key_strings(self): + unsorted = ["Maggie", "Homer", "Bart", "Marge"] + results = [] + for x in unsorted: + insort_left(results, x, key=lambda p: p.lower()) + self.assertEqual(results, ["Bart", "Homer", "Maggie", "Marge"]) + + def test_insort_left_key_dict(self): + unsorted = [{"name":"Maggie"}, {"name":"Homer"}, {"name":"Bart"}, {"name":"Marge"}] + results = [] + for x in unsorted: + insort_left(results, x, key=lambda p: p["name"].lower()) + self.assertEqual(results, [{"name":"Bart"}, {"name":"Homer"}, {"name":"Maggie"}, {"name":"Marge"}])