almalinux/lorax
13
0
Fork 0
Code Issues Pull Requests Releases Activity

lorax-composer: Handle packages with multiple builds

Browse Source 
When the repository has multiple arches, eg. i686 and x86_64, it should
add a new entry to the project's builds list, not create a new project
in the list.

This handles that by adding a modified insort_left function and
examining the packages returned from dnf to make sure they aren't
already listed in the results. It also handles adding them in sorted
order so that no further sorting needs to be done on the results.

Resolves: rhbz#1656642
(cherry picked from commit d18934775c)
This commit is contained in:
Brian C. Lane 2018-12-06 11:13:36 -08:00
parent 574360f7d0
commit 3f034a981d
3 changed files with 160 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/pylorax/api/bisect.py Normal file
View File

@ -0,0 +1,49 @@
#
# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
def insort_left(a, x, key=None, lo=0, hi=None):
"""Insert item x in list a, and keep it sorted assuming a is sorted.
:param a: sorted list
:type a: list
:param x: item to insert into the list
:type x: object
:param key: Function to use to compare items in the list
:type key: function
:returns: index where the item was inserted
:rtype: int
If x is already in a, insert it to the left of the leftmost x.
Optional args lo (default 0) and hi (default len(a)) bound the
slice of a to be searched.
This is a modified version of bisect.insort_left that can use a
function for the compare, and returns the index position where it
was inserted.
"""
if key is None:
key = lambda i: i
if lo < 0:
raise ValueError('lo must be non-negative')
if hi is None:
hi = len(a)
while lo < hi:
mid = (lo+hi)//2
if key(a[mid]) < key(x): lo = mid+1
else: hi = mid
a.insert(lo, x)
return lo

88
src/pylorax/api/projects.py
View File

@ -17,12 +17,14 @@
import logging
log = logging.getLogger("lorax-composer")
import os
from configparser import ConfigParser
import dnf
from glob import glob
import os
import time
from pylorax.api.bisect import insort_left
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
@ -75,6 +77,32 @@ def pkg_to_project(pkg):
"upstream_vcs": "UPSTREAM_VCS"}
def pkg_to_build(pkg):
"""Extract the build details from a hawkey.Package object
:param pkg: hawkey.Package object with package details
:type pkg: hawkey.Package
:returns: A dict with the build details, epoch, release, arch, build_time, changelog, ...
:rtype: dict
metadata entries are hard-coded to {}
Note that this only returns the build dict, it does not include the name, description, etc.
"""
return {"epoch": pkg.epoch,
"release": pkg.release,
"arch": pkg.arch,
"build_time": api_time(pkg.buildtime),
"changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package
"build_config_ref": "BUILD_CONFIG_REF",
"build_env_ref": "BUILD_ENV_REF",
"metadata": {},
"source": {"license": pkg.license,
"version": pkg.version,
"source_ref": "SOURCE_REF",
"metadata": {}}}
def pkg_to_project_info(pkg):
"""Extract the details from a hawkey.Package object
@ -85,25 +113,12 @@ def pkg_to_project_info(pkg):
metadata entries are hard-coded to {}
"""
build = {"epoch": pkg.epoch,
"release": pkg.release,
"arch": pkg.arch,
"build_time": api_time(pkg.buildtime),
"changelog": "CHANGELOG_NEEDED", # XXX Not in hawkey.Package
"build_config_ref": "BUILD_CONFIG_REF",
"build_env_ref": "BUILD_ENV_REF",
"metadata": {},
"source": {"license": pkg.license,
"version": pkg.version,
"source_ref": "SOURCE_REF",
"metadata": {}}}
return {"name": pkg.name,
"summary": pkg.summary,
"description": pkg.description,
"homepage": pkg.url,
"upstream_vcs": "UPSTREAM_VCS",
"builds": [build]}
"builds": [pkg_to_build(pkg)]}
def pkg_to_dep(pkg):
@ -180,7 +195,24 @@ def projects_info(dbo, project_names):
pkgs = dbo.sack.query().available().filter(name__glob=project_names)
else:
pkgs = dbo.sack.query().available()
return sorted(map(pkg_to_project_info, pkgs), key=lambda p: p["name"].lower())
# iterate over pkgs
# - if pkg.name isn't in the results yet, add pkg_to_project_info in sorted position
# - if pkg.name is already in results, get its builds. If the build for pkg is different
# in any way (version, arch, etc.) add it to the entry's builds list. If it is the same,
# skip it.
results = []
results_names = {}
for p in pkgs:
if p.name.lower() not in results_names:
idx = insort_left(results, pkg_to_project_info(p), key=lambda p: p["name"].lower())
results_names[p.name.lower()] = idx
else:
build = pkg_to_build(p)
if build not in results[results_names[p.name.lower()]]["builds"]:
results[results_names[p.name.lower()]]["builds"].append(build)
return results
def _depsolve(dbo, projects, groups):
"""Add projects to a new transaction
@ -321,9 +353,29 @@ def modules_list(dbo, module_names):
"""
# TODO - Figure out what to do with this for Fedora 'modules'
projs = projects_info(dbo, module_names)
return sorted(map(proj_to_module, projs), key=lambda p: p["name"].lower())
projs = _unique_dicts(projects_info(dbo, module_names), key=lambda p: p["name"].lower())
return list(map(proj_to_module, projs))
def _unique_dicts(lst, key):
"""Return a new list of dicts, only including one match of key(d)
:param lst: list of dicts
:type lst: list
:param key: key function to match lst entries
:type key: function
:returns: list of the unique lst entries
:rtype: list
Uses key(d) to test for duplicates in the returned list, creating a
list of unique return values.
"""
result = []
result_keys = []
for d in lst:
if key(d) not in result_keys:
result.append(d)
result_keys.append(key(d))
return result
def modules_info(dbo, module_names):
"""Return details about a module, including dependencies

41
tests/pylorax/test_bisect.py Normal file
View File

@ -0,0 +1,41 @@
#
# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import unittest
from pylorax.api.bisect import insort_left
class BisectTest(unittest.TestCase):
def test_insort_left_nokey(self):
results = []
for x in range(0, 10):
insort_left(results, x)
self.assertEqual(results, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
def test_insort_left_key_strings(self):
unsorted = ["Maggie", "Homer", "Bart", "Marge"]
results = []
for x in unsorted:
insort_left(results, x, key=lambda p: p.lower())
self.assertEqual(results, ["Bart", "Homer", "Maggie", "Marge"])
def test_insort_left_key_dict(self):
unsorted = [{"name":"Maggie"}, {"name":"Homer"}, {"name":"Bart"}, {"name":"Marge"}]
results = []
for x in unsorted:
insort_left(results, x, key=lambda p: p["name"].lower())
self.assertEqual(results, [{"name":"Bart"}, {"name":"Homer"}, {"name":"Maggie"}, {"name":"Marge"}])