ALBS-931 Added integration with AlmaLinux Build System (errata feed) #3

Merged
kzhukov merged 4 commits from ALBS-931 into main 2023-01-20 11:49:11 +00:00
6 changed files with 143 additions and 13 deletions
Showing only changes of commit b2a3e5ac70 - Show all commits

3
.gitignore vendored
View File

@ -3,4 +3,5 @@ logs
results results
*.pyc *.pyc
__pycache__ __pycache__
.vscode .vscode
private*

View File

@ -0,0 +1,72 @@
"""
albs.py contains ALBS class
"""
from typing import Union, Dict
import requests
class ALBS:
"""
ALBS class implemets buildsys.almalinux.org API interaction logic
"""
def __init__(self, url: str, token: str, timeout: int):
if url.endswith('/'):
kzhukov marked this conversation as resolved
Review

It's not necessary if you fix comments below.

It's not necessary if you fix comments below.
url = url[:-1]
self.url = url
self.token = token
self.timeout = timeout
self._platforms = self._get_platforms()
def _get_platforms(self) -> Dict[str, int]:
'''
Getting list of all platforms and
return Dict: platform_name -> platform_id
'''
endpoint = '/api/v1/platforms/'
headers = {'accept': 'application/json',
'Authorization': f'Bearer {self.token}'}
response = requests.get(url=self.url+endpoint,
kzhukov marked this conversation as resolved
Review

self.url+endpoint don't do the same, please :)

from urllib.parse import urljoin
urljoin(url + '/', some_uri)

That code join part of an URL right in all cases

`self.url+endpoint` don't do the same, please :) ``` from urllib.parse import urljoin urljoin(url + '/', some_uri) ``` That code join part of an URL right in all cases
headers=headers,
timeout=self.timeout)
response.raise_for_status()
res = {}

That looks more pretty I guess

res = {platform['name']: platform['id'] for platform in response.json()}
That looks more pretty I guess ``` res = {platform['name']: platform['id'] for platform in response.json()} ```
for platform in response.json():
res[platform['name']] = platform['id']
return res
def get_errata_status(self, errata_id: str, platform_name: str) -> Union[str, None]:
"""
Get release status for particular errata_id
Params
------
errata_id: str: errata id to get (ALSA-2023:0095)
Returns
-------
str: release status
If errata_id was not found Returns None
Raises
------
Any errors raised by requests libary
ValueError if platform_name not found in buildsys
"""
endpoint = '/api/v1/errata/query/'
# platformId
try:
platform_id = self._platforms[platform_name]
except KeyError as error:
raise ValueError(f'{platform_name} was not found') from error
params = {'id': errata_id, 'platformId': platform_id}
headers = {'accept': 'application/json',
'Authorization': f'Bearer {self.token}'}
response = requests.get(url=self.url+endpoint,
kzhukov marked this conversation as resolved
Review

same as above

same as above
params=params, headers=headers,
timeout=self.timeout)
response.raise_for_status()
response_json = response.json()
# errata_id was not found
if response_json['total_records'] == 0:
return None

jfyi, return None is redundunt form of return

jfyi, `return None` is redundunt form of `return`
return response_json['records'][0]['release_status']

View File

@ -1,5 +1,5 @@
""" """
package comparer.py implemets difference checking logic module comparer.py implemets difference checking logic
""" """
import bz2 import bz2
@ -13,9 +13,10 @@ import xml.etree.ElementTree as ET
import requests import requests
from .advisory import Advisory
from .albs import ALBS
from .config import Config from .config import Config
from .package import Package from .package import Package
from .advisory import Advisory
def download_oval(url: str, download_dir: Path) -> str: def download_oval(url: str, download_dir: Path) -> str:
@ -134,11 +135,14 @@ def compare(rhel_oval: Dict[str, Advisory],
alma_oval: Dict[str, Advisory], alma_oval: Dict[str, Advisory],
alma_errata: Dict[str, Advisory], alma_errata: Dict[str, Advisory],
advisory_exclude: List[str], advisory_exclude: List[str],
packages_exclude: List[str]) -> Tuple[dict, list]: packages_exclude: List[str],
albs: ALBS,
release: str) -> Tuple[dict, list]:
""" """
compares rhel oval with alma oval and alma errata compares rhel oval with alma oval and alma errata
""" """
diff = [] diff = []
report = { report = {
# total amount of security advisories # total amount of security advisories
'total_advisory_count': 0, 'total_advisory_count': 0,
@ -169,7 +173,10 @@ def compare(rhel_oval: Dict[str, Advisory],
# total amount of unique missing packages across all alma SA # total amount of unique missing packages across all alma SA
'missing_packages_unique_count': 0, 'missing_packages_unique_count': 0,
# list of unique packages that missing across all alma SA # list of unique packages that missing across all alma SA
'missing_packages_unique': [] 'missing_packages_unique': [],
# contains errata release status from buildsystem
# this list populated for missing advisories only
'miss_adv_albs_errata_release_status': [],
} }
for rhel_advisory_id, rhel_advisory in rhel_oval.items(): for rhel_advisory_id, rhel_advisory in rhel_oval.items():
@ -234,7 +241,8 @@ def compare(rhel_oval: Dict[str, Advisory],
if str(r) not in [str(i) for i in alma_errata_packages]] if str(r) not in [str(i) for i in alma_errata_packages]]
if alma_errata_missing_packages: if alma_errata_missing_packages:
report['diff_count'] += 1 report['diff_count'] += 1
diff_str = f"Errata advisory has missing packages: {','.join(alma_errata_missing_packages)}" mp_string = ','.join(alma_errata_missing_packages)
diff_str = f"Errata advisory has missing packages: {mp_string}"
diff.append({'advisory_name': advisory_name, diff.append({'advisory_name': advisory_name,
'diff': diff_str}) 'diff': diff_str})
report['errata_missing_pkg_advisory'].append(advisory_name) report['errata_missing_pkg_advisory'].append(advisory_name)
@ -247,9 +255,22 @@ def compare(rhel_oval: Dict[str, Advisory],
# if we here, all checks were passed # if we here, all checks were passed
report['good_advisory_count'] += 1 report['good_advisory_count'] += 1
for item in report.values(): # albs errata flow
if isinstance(item, list): logging.info('Getting errata release status for missing advisories')
item.sort() missing_advisories = report['errata_missing_advisory'] + \
report['oval_missing_advisory']
missing_advisories = list(dict.fromkeys(missing_advisories))
for adv in missing_advisories:
try:
release_status = albs.get_errata_status(
adv, f'AlmaLinux-{release}')
except Exception as err: # pylint: disable=broad-except
logging.error("cant get release status for %s: %s", adv, err)
continue
if release_status is None:
release_status = 'not-found-in-errata-flow'
report['miss_adv_albs_errata_release_status'].append(
{"advisory": adv, "release_status": release_status})
return report, diff return report, diff
@ -280,12 +301,17 @@ def comparer_run(config: Config) -> Dict[str, Any]:
alma_errata_dict = parse_errata(alma_errata_file) alma_errata_dict = parse_errata(alma_errata_file)
logging.info('Comparing rhel and alma') logging.info('Comparing rhel and alma')
report_release, diff_release = \ albs = ALBS(config.albs_url,
config.albs_jwt_token,
config.albs_timeout)
report_release, diff_release =\
compare(rhel_oval_dict, compare(rhel_oval_dict,
alma_oval_dict, alma_oval_dict,
alma_errata_dict, alma_errata_dict,
config.advisory_exclude, config.advisory_exclude,
config.packages_exclude) config.packages_exclude,
albs, release)
result[release] = {'report': report_release, result[release] = {'report': report_release,
'diff': diff_release, 'diff': diff_release,
'rhel_oval_url': urls.rhel_oval_url, 'rhel_oval_url': urls.rhel_oval_url,

View File

@ -22,6 +22,8 @@ SERVER_IP = IPv4Address('127.0.0.1')
# not checking anything before RHEL-9.0 release # not checking anything before RHEL-9.0 release
NOT_BEFORE = datetime(2022, 5, 18) NOT_BEFORE = datetime(2022, 5, 18)
UPDATE_INTERVAL_MINUTES = 30 UPDATE_INTERVAL_MINUTES = 30
ALBS_URL = 'https://build.almalinux.org'
ALBS_TIMEOUT = 30
class ReleaseUrls(BaseModel): class ReleaseUrls(BaseModel):
@ -65,6 +67,14 @@ class Config(BaseModel):
update_interval_minutes: int = Field( update_interval_minutes: int = Field(
description='how often service will be running difference checks (in minutes)', description='how often service will be running difference checks (in minutes)',
default=UPDATE_INTERVAL_MINUTES) default=UPDATE_INTERVAL_MINUTES)
albs_url: str = Field(
description='URL of Alma linux build system',
default=ALBS_URL)
albs_jwt_token: str = Field(
description='JWT token that will be used when querying ALBS API')
albs_timeout: int = Field(
description='max time (in seconds) that service will be wait for ALBS API to response',
default=ALBS_TIMEOUT)
@validator("releases", pre=True) @validator("releases", pre=True)
@classmethod @classmethod

View File

@ -65,4 +65,22 @@ not_before: 2022-5-18
# how often service will be running difference checks (in minutes) # how often service will be running difference checks (in minutes)
# required: no # required: no
# default: 30 # default: 30
update_interval_minutes: 30 update_interval_minutes: 30
# albs_url
# URL of Alma linux build system
# required: no
# default: https://build.almalinux.org
albs_url: https://build.almalinux.org
# albs_jwt_token
# JWT token that will be used when querying ALBS API
# required: yes
# default: N/A
albs_jwt_token:
# albs_timeout
# max time (in seconds) that service will be wait for ALBS API to response
# required: no
# default: 30
albs_timeout: 30

View File

@ -3,4 +3,7 @@
2023-01-04 v1.0.1 2023-01-04 v1.0.1
Fixed missing packages false positives Fixed missing packages false positives
2023-01-12 v1.0.2 2023-01-12 v1.0.2
Added support for Bug/Enhancement Advisories Added support for Bug/Enhancement Advisories
2023-01-19 v2.0.0
Added integration with AlmaLinux Build System (errata feed)