Merge pull request 'ALBS-931 Added integration with AlmaLinux Build System (errata feed)' (#3) from ALBS-931 into main

Reviewed-on: #3

.gitignore

@@ -3,4 +3,5 @@ logs
 results
 *.pyc
 __pycache__
-.vscode
+.vscode
+private*

albs_oval_errata_diff/albs.py

@@ -0,0 +1,70 @@
+"""
+albs.py contains ALBS class
+"""
+from urllib.parse import urljoin
+from typing import Union, Dict
+
+import requests
+
+
+class ALBS:
+    """
+    ALBS class implemets buildsys.almalinux.org API interaction logic
+    """
+
+    def __init__(self, url: str, token: str, timeout: int):
+        self.url = url
+        self.token = token
+        self.timeout = timeout
+        self._platforms = self._get_platforms()
+
+    def _get_platforms(self) -> Dict[str, int]:
+        '''
+        Getting list of all platforms and
+        return Dict: platform_name -> platform_id
+        '''
+        endpoint = '/api/v1/platforms/'
+        headers = {'accept': 'application/json',
+                   'Authorization': f'Bearer {self.token}'}
+        response = requests.get(url=urljoin(self.url, endpoint),
+                                headers=headers,
+                                timeout=self.timeout)
+        response.raise_for_status()
+        res = {platform['name']: platform['id']
+               for platform in response.json()}
+        return res
+
+    def get_errata_status(self, errata_id: str, platform_name: str) -> Union[str, None]:
+        """
+        Get release status for particular errata_id
+        Params
+        ------
+            errata_id: str: errata id to get (ALSA-2023:0095)
+        Returns
+        -------
+            str: release status
+            If errata_id was not found Returns None
+        Raises
+        ------
+            Any errors raised by requests libary
+            ValueError if platform_name not found in buildsys
+        """
+        endpoint = '/api/v1/errata/query/'
+        # platformId
+        try:
+            platform_id = self._platforms[platform_name]
+        except KeyError as error:
+            raise ValueError(f'{platform_name} was not found') from error
+        params = {'id': errata_id, 'platformId': platform_id}
+        headers = {'accept': 'application/json',
+                   'Authorization': f'Bearer {self.token}'}
+        response = requests.get(url=urljoin(self.url, endpoint),
+                                params=params, headers=headers,
+                                timeout=self.timeout)
+        response.raise_for_status()
+        response_json = response.json()
+
+        # errata_id was not found
+        if response_json['total_records'] == 0:
+            return
+        return response_json['records'][0]['release_status']

albs_oval_errata_diff/comparer.py

@@ -1,5 +1,5 @@
 """
-package comparer.py implemets difference checking logic
+module comparer.py implemets difference checking logic
 """
 
 import bz2
@@ -13,9 +13,10 @@ import xml.etree.ElementTree as ET
 
 import requests
 
+from .advisory import Advisory
+from .albs import ALBS
 from .config import Config
 from .package import Package
-from .advisory import Advisory
 
 
 def download_oval(url: str, download_dir: Path) -> str:
@@ -134,11 +135,14 @@ def compare(rhel_oval: Dict[str, Advisory],
             alma_oval: Dict[str, Advisory],
             alma_errata: Dict[str, Advisory],
             advisory_exclude: List[str],
-            packages_exclude: List[str]) -> Tuple[dict, list]:
+            packages_exclude: List[str],
+            albs: ALBS,
+            release: str) -> Tuple[dict, list]:
     """
     compares rhel oval with alma oval and alma errata
     """
     diff = []
+
     report = {
         # total amount of security advisories
         'total_advisory_count': 0,
@@ -169,7 +173,10 @@ def compare(rhel_oval: Dict[str, Advisory],
         # total amount of unique missing packages across all alma SA
         'missing_packages_unique_count': 0,
         # list of unique packages that missing across all alma SA
-        'missing_packages_unique': []
+        'missing_packages_unique': [],
+        # contains errata release status from buildsystem
+        # this list populated for missing advisories only
+        'miss_adv_albs_errata_release_status': [],
     }
 
     for rhel_advisory_id, rhel_advisory in rhel_oval.items():
@@ -234,7 +241,8 @@ def compare(rhel_oval: Dict[str, Advisory],
              if str(r) not in [str(i) for i in alma_errata_packages]]
         if alma_errata_missing_packages:
             report['diff_count'] += 1
-            diff_str = f"Errata advisory has missing packages: {','.join(alma_errata_missing_packages)}"
+            mp_string = ','.join(alma_errata_missing_packages)
+            diff_str = f"Errata advisory has missing packages: {mp_string}"
             diff.append({'advisory_name': advisory_name,
                          'diff': diff_str})
             report['errata_missing_pkg_advisory'].append(advisory_name)
@@ -247,9 +255,22 @@ def compare(rhel_oval: Dict[str, Advisory],
             # if we here, all checks were passed
             report['good_advisory_count'] += 1
 
-    for item in report.values():
+    # albs errata flow
-        if isinstance(item, list):
+    logging.info('Getting errata release status for missing advisories')
-            item.sort()
+    missing_advisories = report['errata_missing_advisory'] + \
+        report['oval_missing_advisory']
+    missing_advisories = list(dict.fromkeys(missing_advisories))
+    for adv in missing_advisories:
+        try:
+            release_status = albs.get_errata_status(
+                adv, f'AlmaLinux-{release}')
+        except Exception as err:  # pylint: disable=broad-except
+            logging.error("cant get release status for %s: %s", adv, err)
+            continue
+        if release_status is None:
+            release_status = 'not-found-in-errata-flow'
+        report['miss_adv_albs_errata_release_status'].append(
+            {"advisory": adv, "release_status": release_status})
     return report, diff
 
 
@@ -280,12 +301,17 @@ def comparer_run(config: Config) -> Dict[str, Any]:
         alma_errata_dict = parse_errata(alma_errata_file)
 
         logging.info('Comparing rhel and alma')
-        report_release, diff_release = \
+        albs = ALBS(config.albs_url,
+                    config.albs_jwt_token,
+                    config.albs_timeout)
+        report_release, diff_release =\
             compare(rhel_oval_dict,
                     alma_oval_dict,
                     alma_errata_dict,
                     config.advisory_exclude,
-                    config.packages_exclude)
+                    config.packages_exclude,
+                    albs, release)
+
         result[release] = {'report': report_release,
                            'diff': diff_release,
                            'rhel_oval_url': urls.rhel_oval_url,

albs_oval_errata_diff/config.py

@@ -22,6 +22,8 @@ SERVER_IP = IPv4Address('127.0.0.1')
 # not checking anything before RHEL-9.0 release
 NOT_BEFORE = datetime(2022, 5, 18)
 UPDATE_INTERVAL_MINUTES = 30
+ALBS_URL = 'https://build.almalinux.org'
+ALBS_TIMEOUT = 30
 
 
 class ReleaseUrls(BaseModel):
@@ -65,6 +67,14 @@ class Config(BaseModel):
     update_interval_minutes: int = Field(
         description='how often service will be running difference checks (in minutes)',
         default=UPDATE_INTERVAL_MINUTES)
+    albs_url: str = Field(
+        description='URL of Alma linux build system',
+        default=ALBS_URL)
+    albs_jwt_token: str = Field(
+        description='JWT token that will be used when querying ALBS API')
+    albs_timeout: int = Field(
+        description='max time (in seconds) that service will be wait for ALBS API to response',
+        default=ALBS_TIMEOUT)
 
     @validator("releases", pre=True)
     @classmethod

config.default.yml

@@ -65,4 +65,22 @@ not_before: 2022-5-18
 # how often service will be running difference checks (in minutes)
 # required: no
 # default: 30
-update_interval_minutes: 30
+update_interval_minutes: 30
+
+# albs_url
+# URL of Alma linux build system
+# required: no
+# default: https://build.almalinux.org
+albs_url: https://build.almalinux.org
+
+# albs_jwt_token
+# JWT token that will be used when querying ALBS API
+# required: yes
+# default: N/A
+albs_jwt_token:
+
+# albs_timeout
+# max time (in seconds) that service will be wait for ALBS API to response
+# required: no
+# default: 30
+albs_timeout: 30

releases.txt

@@ -1,6 +1,11 @@
 2022-12-30 v1.0.0
     First version of service
+
 2023-01-04 v1.0.1
     Fixed missing packages false positives
+
 2023-01-12 v1.0.2
-    Added support for Bug/Enhancement Advisories
+    Added support for Bug/Enhancement Advisories
+
+2023-01-20 v2.0.0
+    Added integration with AlmaLinux Build System (errata feed)