diff --git a/src/pylorax/api/bisect.py b/src/pylorax/api/bisect.py
new file mode 100644
index 00000000..a1bbdcc0
--- /dev/null
+++ b/src/pylorax/api/bisect.py
@@ -0,0 +1,49 @@
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+def insort_left(a, x, key=None, lo=0, hi=None):
+ """Insert item x in list a, and keep it sorted assuming a is sorted.
+
+ :param a: sorted list
+ :type a: list
+ :param x: item to insert into the list
+ :type x: object
+ :param key: Function to use to compare items in the list
+ :type key: function
+ :returns: index where the item was inserted
+ :rtype: int
+
+ If x is already in a, insert it to the left of the leftmost x.
+ Optional args lo (default 0) and hi (default len(a)) bound the
+ slice of a to be searched.
+
+ This is a modified version of bisect.insort_left that can use a
+ function for the compare, and returns the index position where it
+ was inserted.
+ """
+ if key is None:
+ key = lambda i: i
+
+ if lo < 0:
+ raise ValueError('lo must be non-negative')
+ if hi is None:
+ hi = len(a)
+ while lo < hi:
+ mid = (lo+hi)//2
+ if key(a[mid]) < key(x): lo = mid+1
+ else: hi = mid
+ a.insert(lo, x)
+ return lo
diff --git a/src/pylorax/api/projects.py b/src/pylorax/api/projects.py
index 0fa1a75c..6b30c27d 100644
--- a/src/pylorax/api/projects.py
+++ b/src/pylorax/api/projects.py
@@ -17,13 +17,14 @@
import logging
log = logging.getLogger("lorax-composer")
-import os
from ConfigParser import ConfigParser
import fnmatch
from glob import glob
+import os
import time
from yum.Errors import YumBaseError
+from pylorax.api.bisect import insort_left
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
@@ -77,6 +78,32 @@ def yaps_to_project(yaps):
"upstream_vcs": "UPSTREAM_VCS"}
+def yaps_to_build(yaps):
+ """Extract the build details from a hawkey.Package object
+
+ :param yaps: Yum object with package details
+ :type yaps: YumAvailablePackageSqlite
+ :returns: A dict with the build details, epoch, release, arch, build_time, changelog, ...
+ :rtype: dict
+
+ metadata entries are hard-coded to {}
+
+ Note that this only returns the build dict, it does not include the name, description, etc.
+ """
+ return {"epoch": int(yaps.epoch),
+ "release": yaps.release,
+ "arch": yaps.arch,
+ "build_time": api_time(yaps.buildtime),
+ "changelog": api_changelog(yaps.returnChangelog()),
+ "build_config_ref": "BUILD_CONFIG_REF",
+ "build_env_ref": "BUILD_ENV_REF",
+ "metadata": {},
+ "source": {"license": yaps.license,
+ "version": yaps.version,
+ "source_ref": "SOURCE_REF",
+ "metadata": {}}}
+
+
def yaps_to_project_info(yaps):
"""Extract the details from a YumAvailablePackageSqlite object
@@ -87,25 +114,12 @@ def yaps_to_project_info(yaps):
metadata entries are hard-coded to {}
"""
- build = {"epoch": int(yaps.epoch),
- "release": yaps.release,
- "arch": yaps.arch,
- "build_time": api_time(yaps.buildtime),
- "changelog": api_changelog(yaps.returnChangelog()),
- "build_config_ref": "BUILD_CONFIG_REF",
- "build_env_ref": "BUILD_ENV_REF",
- "metadata": {},
- "source": {"license": yaps.license,
- "version": yaps.version,
- "source_ref": "SOURCE_REF",
- "metadata": {}}}
-
return {"name": yaps.name,
"summary": yaps.summary,
"description": yaps.description,
"homepage": yaps.url,
"upstream_vcs": "UPSTREAM_VCS",
- "builds": [build]}
+ "builds": [yaps_to_build(yaps)]}
def tm_to_dep(tm):
@@ -170,7 +184,6 @@ def projects_list(yb):
yb.closeRpmDB()
return sorted(map(yaps_to_project, ybl.available), key=lambda p: p["name"].lower())
-
def projects_info(yb, project_names):
"""Return details about specific projects
@@ -187,7 +200,25 @@ def projects_info(yb, project_names):
raise ProjectsError("There was a problem with info for %s: %s" % (project_names, str(e)))
finally:
yb.closeRpmDB()
- return sorted(map(yaps_to_project_info, ybl.available), key=lambda p: p["name"].lower())
+
+ # iterate over pkgs
+ # - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position
+ # - if pkg.name is already in results, get its builds. If the build for pkg is different
+ # in any way (version, arch, etc.) add it to the entry's builds list. If it is the same,
+ # skip it.
+ results = []
+ results_names = {}
+ for p in ybl.available:
+ if p.name.lower() not in results_names:
+ idx = insort_left(results, yaps_to_project_info(p), key=lambda p: p["name"].lower())
+ results_names[p.name.lower()] = idx
+ else:
+ build = yaps_to_build(p)
+ if build not in results[results_names[p.name.lower()]]["builds"]:
+ results[results_names[p.name.lower()]]["builds"].append(build)
+
+ return results
+
def filterVersionGlob(pkgs, version):
"""Filter a list of yum package objects with a version glob
@@ -373,14 +404,29 @@ def modules_list(yb, module_names):
Modules don't exist in RHEL7 so this only returns projects
and sets the type to "rpm"
"""
- try:
- ybl = yb.doPackageLists(pkgnarrow="available", patterns=module_names, showdups=False)
- except YumBaseError as e:
- raise ProjectsError("There was a problem listing modules: %s" % str(e))
- finally:
- yb.closeRpmDB()
- return sorted(map(yaps_to_module, ybl.available), key=lambda p: p["name"].lower())
+ projs = _unique_dicts(projects_info(yb, module_names), key=lambda p: p["name"].lower())
+ return list(map(yaps_to_module, projs))
+def _unique_dicts(lst, key):
+ """Return a new list of dicts, only including one match of key(d)
+
+ :param lst: list of dicts
+ :type lst: list
+ :param key: key function to match lst entries
+ :type key: function
+ :returns: list of the unique lst entries
+ :rtype: list
+
+ Uses key(d) to test for duplicates in the returned list, creating a
+ list of unique return values.
+ """
+ result = []
+ result_keys = []
+ for d in lst:
+ if key(d) not in result_keys:
+ result.append(d)
+ result_keys.append(key(d))
+ return result
def modules_info(yb, module_names):
"""Return details about a module, including dependencies
diff --git a/tests/pylorax/test_bisect.py b/tests/pylorax/test_bisect.py
new file mode 100644
index 00000000..8e5a101a
--- /dev/null
+++ b/tests/pylorax/test_bisect.py
@@ -0,0 +1,41 @@
+#
+# Copyright (C) 2018 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+import unittest
+
+from pylorax.api.bisect import insort_left
+
+
+class BisectTest(unittest.TestCase):
+ def test_insort_left_nokey(self):
+ results = []
+ for x in range(0, 10):
+ insort_left(results, x)
+ self.assertEqual(results, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+ def test_insort_left_key_strings(self):
+ unsorted = ["Maggie", "Homer", "Bart", "Marge"]
+ results = []
+ for x in unsorted:
+ insort_left(results, x, key=lambda p: p.lower())
+ self.assertEqual(results, ["Bart", "Homer", "Maggie", "Marge"])
+
+ def test_insort_left_key_dict(self):
+ unsorted = [{"name":"Maggie"}, {"name":"Homer"}, {"name":"Bart"}, {"name":"Marge"}]
+ results = []
+ for x in unsorted:
+ insort_left(results, x, key=lambda p: p["name"].lower())
+ self.assertEqual(results, [{"name":"Bart"}, {"name":"Homer"}, {"name":"Maggie"}, {"name":"Marge"}])