lorax-composer: Handle packages with multiple builds

When the repository has multiple arches, eg. i686 and x86_64, it should
add a new entry to the project's builds list, not create a new project
in the list.

This handles that by adding a modified insort_left function and
examining the packages returned from dnf to make sure they aren't
already listed in the results. It also handles adding them in sorted
order so that no further sorting needs to be done on the results.

Resolves: rhbz#1656642
(cherry picked from commit d18934775c)

src/pylorax/api/bisect.py

@@ -0,0 +1,49 @@
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+def insort_left(a, x, key=None, lo=0, hi=None):
+    """Insert item x in list a, and keep it sorted assuming a is sorted.
+
+    :param a: sorted list
+    :type a: list
+    :param x: item to insert into the list
+    :type x: object
+    :param key: Function to use to compare items in the list
+    :type key: function
+    :returns: index where the item was inserted
+    :rtype: int
+
+    If x is already in a, insert it to the left of the leftmost x.
+    Optional args lo (default 0) and hi (default len(a)) bound the
+    slice of a to be searched.
+
+    This is a modified version of bisect.insort_left that can use a
+    function for the compare, and returns the index position where it
+    was inserted.
+    """
+    if key is None:
+        key = lambda i: i
+
+    if lo < 0:
+        raise ValueError('lo must be non-negative')
+    if hi is None:
+        hi = len(a)
+    while lo < hi:
+        mid = (lo+hi)//2
+        if key(a[mid]) < key(x): lo = mid+1
+        else: hi = mid
+    a.insert(lo, x)
+    return lo

src/pylorax/api/projects.py

@@ -17,12 +17,14 @@
 import logging
 log = logging.getLogger("lorax-composer")
 
-import os
 from configparser import ConfigParser
 import dnf
 from glob import glob
+import os
 import time
 
+from pylorax.api.bisect import insort_left
+
 TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
 
 
@@ -75,17 +77,19 @@ def pkg_to_project(pkg):
             "upstream_vcs": "UPSTREAM_VCS"}
 
 
-def pkg_to_project_info(pkg):
+def pkg_to_build(pkg):
-    """Extract the details from a hawkey.Package object
+    """Extract the build details from a hawkey.Package object
 
     :param pkg: hawkey.Package object with package details
     :type pkg: hawkey.Package
-    :returns: A dict with the project details, as well as epoch, release, arch, build_time, changelog, ...
+    :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ...
     :rtype: dict
 
     metadata entries are hard-coded to {}
+
+    Note that this only returns the build dict, it does not include the name, description, etc.
     """
-    build = {"epoch":      pkg.epoch,
+    return {"epoch":      pkg.epoch,
             "release":    pkg.release,
             "arch":       pkg.arch,
             "build_time": api_time(pkg.buildtime),
@@ -98,12 +102,23 @@ def pkg_to_project_info(pkg):
                             "source_ref": "SOURCE_REF",
                             "metadata":   {}}}
 
+
+def pkg_to_project_info(pkg):
+    """Extract the details from a hawkey.Package object
+
+    :param pkg: hawkey.Package object with package details
+    :type pkg: hawkey.Package
+    :returns: A dict with the project details, as well as epoch, release, arch, build_time, changelog, ...
+    :rtype: dict
+
+    metadata entries are hard-coded to {}
+    """
     return {"name":         pkg.name,
             "summary":      pkg.summary,
             "description":  pkg.description,
             "homepage":     pkg.url,
             "upstream_vcs": "UPSTREAM_VCS",
-            "builds":       [build]}
+            "builds":       [pkg_to_build(pkg)]}
 
 
 def pkg_to_dep(pkg):
@@ -180,7 +195,24 @@ def projects_info(dbo, project_names):
         pkgs = dbo.sack.query().available().filter(name__glob=project_names)
     else:
         pkgs = dbo.sack.query().available()
-    return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
+
+    # iterate over pkgs
+    # - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position
+    # - if pkg.name is already in results, get its builds. If the build for pkg is different
+    #   in any way (version, arch, etc.) add it to the entry's builds list. If it is the same,
+    #   skip it.
+    results = []
+    results_names = {}
+    for p in pkgs:
+        if p.name.lower() not in results_names:
+            idx = insort_left(results, pkg_to_project_info(p), key=lambda p: p["name"].lower())
+            results_names[p.name.lower()] = idx
+        else:
+            build = pkg_to_build(p)
+            if build not in results[results_names[p.name.lower()]]["builds"]:
+                results[results_names[p.name.lower()]]["builds"].append(build)
+
+    return results
 
 def _depsolve(dbo, projects, groups):
     """Add projects to a new transaction
@@ -319,9 +351,29 @@ def modules_list(dbo, module_names):
 
     """
     # TODO - Figure out what to do with this for Fedora 'modules'
-    projs = projects_info(dbo, module_names)
+    projs = _unique_dicts(projects_info(dbo, module_names), key=lambda p: p["name"].lower())
-    return sorted(map(proj_to_module, projs), key=lambda p: p["name"].lower())
+    return list(map(proj_to_module, projs))
 
+def _unique_dicts(lst, key):
+    """Return a new list of dicts, only including one match of key(d)
+
+    :param lst: list of dicts
+    :type lst: list
+    :param key: key function to match lst entries
+    :type key: function
+    :returns: list of the unique lst entries
+    :rtype: list
+
+    Uses key(d) to test for duplicates in the returned list, creating a
+    list of unique return values.
+    """
+    result = []
+    result_keys = []
+    for d in lst:
+        if key(d) not in result_keys:
+            result.append(d)
+            result_keys.append(key(d))
+    return result
 
 def modules_info(dbo, module_names):
     """Return details about a module, including dependencies

tests/pylorax/test_bisect.py

@@ -0,0 +1,41 @@
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+import unittest
+
+from pylorax.api.bisect import insort_left
+
+
+class BisectTest(unittest.TestCase):
+    def test_insort_left_nokey(self):
+        results = []
+        for x in range(0, 10):
+            insort_left(results, x)
+        self.assertEqual(results, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+    def test_insort_left_key_strings(self):
+        unsorted = ["Maggie", "Homer", "Bart", "Marge"]
+        results = []
+        for x in unsorted:
+            insort_left(results, x, key=lambda p: p.lower())
+        self.assertEqual(results, ["Bart", "Homer", "Maggie", "Marge"])
+
+    def test_insort_left_key_dict(self):
+        unsorted = [{"name":"Maggie"}, {"name":"Homer"}, {"name":"Bart"}, {"name":"Marge"}]
+        results = []
+        for x in unsorted:
+            insort_left(results, x, key=lambda p: p["name"].lower())
+        self.assertEqual(results, [{"name":"Bart"}, {"name":"Homer"}, {"name":"Maggie"}, {"name":"Marge"}])