From b2a3e5ac7083ff39652ae85e4ca2dc232d1951ae Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 19 Jan 2023 13:17:42 +0100 Subject: [PATCH 1/4] ALBS-931 Added integration with AlmaLinux Build System (errata feed) --- .gitignore | 3 +- albs_oval_errata_diff/albs.py | 72 +++++++++++++++++++++++++++++++ albs_oval_errata_diff/comparer.py | 46 +++++++++++++++----- albs_oval_errata_diff/config.py | 10 +++++ config.default.yml | 20 ++++++++- releases.txt | 5 ++- 6 files changed, 143 insertions(+), 13 deletions(-) create mode 100644 albs_oval_errata_diff/albs.py diff --git a/.gitignore b/.gitignore index 886279f..bf7862e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ logs results *.pyc __pycache__ -.vscode \ No newline at end of file +.vscode +private* \ No newline at end of file diff --git a/albs_oval_errata_diff/albs.py b/albs_oval_errata_diff/albs.py new file mode 100644 index 0000000..da2b718 --- /dev/null +++ b/albs_oval_errata_diff/albs.py @@ -0,0 +1,72 @@ +""" +albs.py contains ALBS class +""" +from typing import Union, Dict + +import requests + + +class ALBS: + """ + ALBS class implemets buildsys.almalinux.org API interaction logic + """ + + def __init__(self, url: str, token: str, timeout: int): + if url.endswith('/'): + url = url[:-1] + self.url = url + self.token = token + self.timeout = timeout + self._platforms = self._get_platforms() + + def _get_platforms(self) -> Dict[str, int]: + ''' + Getting list of all platforms and + return Dict: platform_name -> platform_id + ''' + endpoint = '/api/v1/platforms/' + headers = {'accept': 'application/json', + 'Authorization': f'Bearer {self.token}'} + response = requests.get(url=self.url+endpoint, + headers=headers, + timeout=self.timeout) + response.raise_for_status() + res = {} + for platform in response.json(): + res[platform['name']] = platform['id'] + return res + + def get_errata_status(self, errata_id: str, platform_name: str) -> Union[str, None]: + """ + Get release status for particular errata_id + Params + ------ + errata_id: str: errata id to get (ALSA-2023:0095) + Returns + ------- + str: release status + If errata_id was not found Returns None + Raises + ------ + Any errors raised by requests libary + ValueError if platform_name not found in buildsys + """ + endpoint = '/api/v1/errata/query/' + # platformId + try: + platform_id = self._platforms[platform_name] + except KeyError as error: + raise ValueError(f'{platform_name} was not found') from error + params = {'id': errata_id, 'platformId': platform_id} + headers = {'accept': 'application/json', + 'Authorization': f'Bearer {self.token}'} + response = requests.get(url=self.url+endpoint, + params=params, headers=headers, + timeout=self.timeout) + response.raise_for_status() + response_json = response.json() + + # errata_id was not found + if response_json['total_records'] == 0: + return None + return response_json['records'][0]['release_status'] diff --git a/albs_oval_errata_diff/comparer.py b/albs_oval_errata_diff/comparer.py index 5809e3f..03765fe 100644 --- a/albs_oval_errata_diff/comparer.py +++ b/albs_oval_errata_diff/comparer.py @@ -1,5 +1,5 @@ """ -package comparer.py implemets difference checking logic +module comparer.py implemets difference checking logic """ import bz2 @@ -13,9 +13,10 @@ import xml.etree.ElementTree as ET import requests +from .advisory import Advisory +from .albs import ALBS from .config import Config from .package import Package -from .advisory import Advisory def download_oval(url: str, download_dir: Path) -> str: @@ -134,11 +135,14 @@ def compare(rhel_oval: Dict[str, Advisory], alma_oval: Dict[str, Advisory], alma_errata: Dict[str, Advisory], advisory_exclude: List[str], - packages_exclude: List[str]) -> Tuple[dict, list]: + packages_exclude: List[str], + albs: ALBS, + release: str) -> Tuple[dict, list]: """ compares rhel oval with alma oval and alma errata """ diff = [] + report = { # total amount of security advisories 'total_advisory_count': 0, @@ -169,7 +173,10 @@ def compare(rhel_oval: Dict[str, Advisory], # total amount of unique missing packages across all alma SA 'missing_packages_unique_count': 0, # list of unique packages that missing across all alma SA - 'missing_packages_unique': [] + 'missing_packages_unique': [], + # contains errata release status from buildsystem + # this list populated for missing advisories only + 'miss_adv_albs_errata_release_status': [], } for rhel_advisory_id, rhel_advisory in rhel_oval.items(): @@ -234,7 +241,8 @@ def compare(rhel_oval: Dict[str, Advisory], if str(r) not in [str(i) for i in alma_errata_packages]] if alma_errata_missing_packages: report['diff_count'] += 1 - diff_str = f"Errata advisory has missing packages: {','.join(alma_errata_missing_packages)}" + mp_string = ','.join(alma_errata_missing_packages) + diff_str = f"Errata advisory has missing packages: {mp_string}" diff.append({'advisory_name': advisory_name, 'diff': diff_str}) report['errata_missing_pkg_advisory'].append(advisory_name) @@ -247,9 +255,22 @@ def compare(rhel_oval: Dict[str, Advisory], # if we here, all checks were passed report['good_advisory_count'] += 1 - for item in report.values(): - if isinstance(item, list): - item.sort() + # albs errata flow + logging.info('Getting errata release status for missing advisories') + missing_advisories = report['errata_missing_advisory'] + \ + report['oval_missing_advisory'] + missing_advisories = list(dict.fromkeys(missing_advisories)) + for adv in missing_advisories: + try: + release_status = albs.get_errata_status( + adv, f'AlmaLinux-{release}') + except Exception as err: # pylint: disable=broad-except + logging.error("cant get release status for %s: %s", adv, err) + continue + if release_status is None: + release_status = 'not-found-in-errata-flow' + report['miss_adv_albs_errata_release_status'].append( + {"advisory": adv, "release_status": release_status}) return report, diff @@ -280,12 +301,17 @@ def comparer_run(config: Config) -> Dict[str, Any]: alma_errata_dict = parse_errata(alma_errata_file) logging.info('Comparing rhel and alma') - report_release, diff_release = \ + albs = ALBS(config.albs_url, + config.albs_jwt_token, + config.albs_timeout) + report_release, diff_release =\ compare(rhel_oval_dict, alma_oval_dict, alma_errata_dict, config.advisory_exclude, - config.packages_exclude) + config.packages_exclude, + albs, release) + result[release] = {'report': report_release, 'diff': diff_release, 'rhel_oval_url': urls.rhel_oval_url, diff --git a/albs_oval_errata_diff/config.py b/albs_oval_errata_diff/config.py index eb7fd71..412f751 100644 --- a/albs_oval_errata_diff/config.py +++ b/albs_oval_errata_diff/config.py @@ -22,6 +22,8 @@ SERVER_IP = IPv4Address('127.0.0.1') # not checking anything before RHEL-9.0 release NOT_BEFORE = datetime(2022, 5, 18) UPDATE_INTERVAL_MINUTES = 30 +ALBS_URL = 'https://build.almalinux.org' +ALBS_TIMEOUT = 30 class ReleaseUrls(BaseModel): @@ -65,6 +67,14 @@ class Config(BaseModel): update_interval_minutes: int = Field( description='how often service will be running difference checks (in minutes)', default=UPDATE_INTERVAL_MINUTES) + albs_url: str = Field( + description='URL of Alma linux build system', + default=ALBS_URL) + albs_jwt_token: str = Field( + description='JWT token that will be used when querying ALBS API') + albs_timeout: int = Field( + description='max time (in seconds) that service will be wait for ALBS API to response', + default=ALBS_TIMEOUT) @validator("releases", pre=True) @classmethod diff --git a/config.default.yml b/config.default.yml index 8c7a7dd..ea298dd 100644 --- a/config.default.yml +++ b/config.default.yml @@ -65,4 +65,22 @@ not_before: 2022-5-18 # how often service will be running difference checks (in minutes) # required: no # default: 30 -update_interval_minutes: 30 \ No newline at end of file +update_interval_minutes: 30 + +# albs_url +# URL of Alma linux build system +# required: no +# default: https://build.almalinux.org +albs_url: https://build.almalinux.org + +# albs_jwt_token +# JWT token that will be used when querying ALBS API +# required: yes +# default: N/A +albs_jwt_token: + +# albs_timeout +# max time (in seconds) that service will be wait for ALBS API to response +# required: no +# default: 30 +albs_timeout: 30 \ No newline at end of file diff --git a/releases.txt b/releases.txt index da18a8c..08ece92 100644 --- a/releases.txt +++ b/releases.txt @@ -3,4 +3,7 @@ 2023-01-04 v1.0.1 Fixed missing packages false positives 2023-01-12 v1.0.2 - Added support for Bug/Enhancement Advisories \ No newline at end of file + Added support for Bug/Enhancement Advisories + +2023-01-19 v2.0.0 + Added integration with AlmaLinux Build System (errata feed) \ No newline at end of file From 23ba530ada77e4169f740e416bc5b41f2255c27b Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 19 Jan 2023 22:13:04 +0100 Subject: [PATCH 2/4] Post-review fixes --- albs_oval_errata_diff/albs.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/albs_oval_errata_diff/albs.py b/albs_oval_errata_diff/albs.py index da2b718..6f758b3 100644 --- a/albs_oval_errata_diff/albs.py +++ b/albs_oval_errata_diff/albs.py @@ -1,6 +1,7 @@ """ albs.py contains ALBS class """ +from urllib.parse import urljoin from typing import Union, Dict import requests @@ -12,8 +13,6 @@ class ALBS: """ def __init__(self, url: str, token: str, timeout: int): - if url.endswith('/'): - url = url[:-1] self.url = url self.token = token self.timeout = timeout @@ -27,13 +26,12 @@ class ALBS: endpoint = '/api/v1/platforms/' headers = {'accept': 'application/json', 'Authorization': f'Bearer {self.token}'} - response = requests.get(url=self.url+endpoint, + response = requests.get(url=urljoin(self.url, endpoint), headers=headers, timeout=self.timeout) response.raise_for_status() - res = {} - for platform in response.json(): - res[platform['name']] = platform['id'] + res = {platform['name']: platform['id'] + for platform in response.json()} return res def get_errata_status(self, errata_id: str, platform_name: str) -> Union[str, None]: @@ -60,7 +58,7 @@ class ALBS: params = {'id': errata_id, 'platformId': platform_id} headers = {'accept': 'application/json', 'Authorization': f'Bearer {self.token}'} - response = requests.get(url=self.url+endpoint, + response = requests.get(url=urljoin(self.url, endpoint), params=params, headers=headers, timeout=self.timeout) response.raise_for_status() @@ -68,5 +66,5 @@ class ALBS: # errata_id was not found if response_json['total_records'] == 0: - return None + return return response_json['records'][0]['release_status'] From ca257a5feff92f5bdb9bcf1c3e12697fe722ebce Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 19 Jan 2023 22:15:42 +0100 Subject: [PATCH 3/4] added spacing in releases.txt --- releases.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/releases.txt b/releases.txt index 08ece92..402791c 100644 --- a/releases.txt +++ b/releases.txt @@ -1,7 +1,9 @@ 2022-12-30 v1.0.0 First version of service + 2023-01-04 v1.0.1 Fixed missing packages false positives + 2023-01-12 v1.0.2 Added support for Bug/Enhancement Advisories From 21c41bb19c3a22b98102055ded5d805a74d66529 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 20 Jan 2023 12:41:37 +0100 Subject: [PATCH 4/4] Updated release date --- releases.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/releases.txt b/releases.txt index 402791c..d913caa 100644 --- a/releases.txt +++ b/releases.txt @@ -7,5 +7,5 @@ 2023-01-12 v1.0.2 Added support for Bug/Enhancement Advisories -2023-01-19 v2.0.0 +2023-01-20 v2.0.0 Added integration with AlmaLinux Build System (errata feed) \ No newline at end of file