From 798f7e4b65d54ef88d43fbe4615ec4bf55ec8c4f Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 28 Dec 2022 17:21:40 +0100 Subject: [PATCH 01/11] Added basic functionality --- .gitignore | 5 + albs-oval-errata-diff.py | 4 + albs_oval_erratta_diff/__init__.py | 0 albs_oval_erratta_diff/comparer.py | 254 +++++++++++++++++++++++++++++ albs_oval_erratta_diff/config.py | 21 +++ albs_oval_erratta_diff/package.py | 14 ++ albs_oval_erratta_diff/sa.py | 15 ++ albs_oval_erratta_diff/start.py | 98 +++++++++++ 8 files changed, 411 insertions(+) create mode 100644 .gitignore create mode 100644 albs-oval-errata-diff.py create mode 100644 albs_oval_erratta_diff/__init__.py create mode 100644 albs_oval_erratta_diff/comparer.py create mode 100644 albs_oval_erratta_diff/config.py create mode 100644 albs_oval_erratta_diff/package.py create mode 100644 albs_oval_erratta_diff/sa.py create mode 100644 albs_oval_erratta_diff/start.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..27ace47 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv +logs +results +*.pyc +__pycache__ \ No newline at end of file diff --git a/albs-oval-errata-diff.py b/albs-oval-errata-diff.py new file mode 100644 index 0000000..8c59f90 --- /dev/null +++ b/albs-oval-errata-diff.py @@ -0,0 +1,4 @@ +from albs_oval_erratta_diff.start import start + + +start() diff --git a/albs_oval_erratta_diff/__init__.py b/albs_oval_erratta_diff/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/albs_oval_erratta_diff/comparer.py b/albs_oval_erratta_diff/comparer.py new file mode 100644 index 0000000..3d96971 --- /dev/null +++ b/albs_oval_erratta_diff/comparer.py @@ -0,0 +1,254 @@ +import bz2 +import datetime +import re +import requests +from typing import Tuple, List, Dict, Any +import xml.etree.ElementTree as ET +import logging +import json + +from .config import DOWNLOAD_DIR, NOT_BEFORE, RELEASES, SA_EXCLUDE, PACKAGES_EXCLUDE +from .sa import SecurityAdvisory +from .package import Package + + +def download_oval(url: str) -> str: + """ + download_oval downloads, decompreses oval file + and returns filepath of saved file + """ + r = requests.get(url, stream=True, timeout=30) + decompressor = bz2.BZ2Decompressor() + fname = url.split('/')[-1].replace('.bz2', '') + fpath = DOWNLOAD_DIR / fname + with open(fpath, 'wb') as fd: + for chunk in r.iter_content(chunk_size=128): + fd.write(decompressor.decompress(chunk)) + return fpath + + +def download_errata(url: str, release_version: int) -> str: + """ + downloads errata_full.json file end returns file path + """ + response = requests.get(url, stream=True, timeout=30) + fname = f'alma-{release_version}.json' + fpath = DOWNLOAD_DIR / fname + with open(fpath, 'wb') as errata_file: + for chunk in response.iter_content(chunk_size=128): + errata_file.write(chunk) + return fpath + + +def parse_oval(fpath: str) -> Dict[str, SecurityAdvisory]: + """ + converting oval xml file to dict + """ + + def extract_package(title: str) -> Package: + r = r'(.*) is earlier than \d+:(.+?(?=-))' + res = re.search(r, title) + name = res.group(1) + version = res.group(2) + return Package(name=name, version=version) + + def extract_id(title: str) -> str: + r = r'[RH|AL]SA-(\d{4}:\d+)(.*)' + res = re.search(r, title) + return res.group(1) + + tree = ET.parse(fpath) + root = tree.getroot() + ns = { + 'n': 'http://oval.mitre.org/XMLSchema/oval-definitions-5', + + } + res = {} + for definition in root.findall('n:definitions/', ns): + title = definition.find('n:metadata/n:title', ns).text + issued = definition.find( + 'n:metadata/n:advisory/n:issued', ns).attrib['date'] + issued_dt = datetime.datetime.strptime(issued, "%Y-%m-%d") + + # we are only interesed in Security advisories after RHEL 8.3 + if ('RHSA' not in title and 'ALSA' not in title) or issued_dt < NOT_BEFORE: + continue + sa_id = extract_id(title) + packages = [extract_package(i.attrib['comment']) for i in definition.findall(".//n:criterion", ns) + if 'is earlier than' in i.attrib['comment']] + res[sa_id] = SecurityAdvisory( + title=title, id=sa_id, packages=packages) + return res + + +def parse_errata(fpath: str) -> Dict[str, SecurityAdvisory]: + """ + parses alma errata file and converts it to dict of SA instances + """ + with open(fpath, 'r', encoding='utf-8') as file_to_load: + erratas = json.load(file_to_load) + res = {} + for errata in erratas['data']: + title = errata['title'] + sa_id = errata['id'].split('-')[-1] + packages = [] + for package in errata['packages']: + full_name = f"{package['name']}-{package['version']}" + if full_name not in packages: + packages.append(full_name) + packages.sort() + res[sa_id] = SecurityAdvisory( + title=title, id=sa_id, packages=packages) + return res + + +def compare(rhel_oval: Dict[str, SecurityAdvisory], + alma_oval: Dict[str, SecurityAdvisory], + alma_errata: Dict[str, SecurityAdvisory]) -> Tuple[dict, list]: + """ + compares rhel oval with alma oval and alma errata + """ + diff = [] + report = { + # total amount of security advisories + 'total_sa_count': 0, + # amount of SA that match with rhel + 'good_sa_count': 0, + # total amount of differencies + 'diff_count': 0, + # list of SA excluded from diff check + 'excluded_sa': [], + # list of packages excluded from diff check + 'excluded_pkg': [], + # amount of oval SA that dont exists in oval file + 'oval_missing_sa_count': 0, + # amount of oval SA that have missing packages + 'oval_missing_pkg_sa_count': 0, + # list of missing oval SA + 'oval_missing_sa': [], + # list of oval SA that have missing packages + 'oval_missing_pkg_sa': [], + # amount of SA that dont exists in errata file + 'errata_missing_sa_count': 0, + # amount of errata SA that have missing packages + 'errata_missing_pkg_sa_count': 0, + # list of SA that are missing in errata file + 'errata_missing_sa': [], + # list of errata SA with missing packages + 'errata_missing_pkg_sa': [], + # total amount of unique missing packages across all alma SA + 'missing_packages_unique_count': 0, + # list of unique packages that missing across all alma SA + 'missing_packages_unique': [] + } + + for rhel_sa_id, rhel_sa in rhel_oval.items(): + report['total_sa_count'] += 1 + sa_name = f'ALSA-{rhel_sa_id}' + + # filtering out SA + if sa_name in SA_EXCLUDE: + report['excluded_sa'].append(sa_name) + continue + + # filtefing out packages + packages_to_check: List[Package] = [] + for p in rhel_sa.packages: + if any(p.name == i for i in PACKAGES_EXCLUDE): + if str(p) not in report['excluded_pkg']: + report['excluded_pkg'].append(str(p)) + else: + packages_to_check.append(p) + + # check oval + try: + alma_oval_sa = alma_oval[rhel_sa_id] + except KeyError: + report['diff_count'] += 1 + diff.append({'sa_name': sa_name, 'diff': 'SA is missing in oval'}) + report['oval_missing_sa'].append(sa_name) + report['oval_missing_sa_count'] += 1 + else: + # check if some packages are missing from oval SA + alma_oval_packages = alma_oval_sa.packages + alma_oval_missing_packages = [str(r) for r in packages_to_check + if r not in alma_oval_packages] + if alma_oval_missing_packages: + report['diff_count'] += 1 + diff.append({'sa_name': sa_name, + 'diff': f"missing packages in oval SA: {','.join(alma_oval_missing_packages)}"}) + report['oval_missing_pkg_sa'].append(sa_name) + report['oval_missing_pkg_sa_count'] += 1 + for mp in alma_oval_missing_packages: + if mp not in report['missing_packages_unique']: + report['missing_packages_unique'].append(mp) + report['missing_packages_unique_count'] += 1 + + # check errata + try: + alma_errata_sa = alma_errata[rhel_sa_id] + except KeyError: + report['errata_missing_sa'].append(sa_name) + report['errata_missing_sa_count'] += 1 + report['diff_count'] += 1 + diff.append( + {'sa_name': sa_name, 'diff': 'SA is missing in errata'}) + continue + # check if some packages are missing from errata SA + alma_errata_packages = alma_errata_sa.packages + alma_errata_missing_packages = [ + str(r) for r in packages_to_check if r not in alma_errata_packages] + if alma_errata_missing_packages: + report['diff_count'] += 1 + diff.append({'sa_name': sa_name, + 'diff': f"missing packages in errata SA: {','.join(alma_errata_missing_packages)}"}) + report['errata_missing_pkg_sa'].append(sa_name) + report['errata_missing_pkg_sa_count'] += 1 + for mp in alma_errata_missing_packages: + if mp not in report['missing_packages_unique']: + report['missing_packages_unique'].append(mp) + report['missing_packages_unique_count'] += 1 + else: + # if we here, all checks were passed + report['good_sa_count'] += 1 + + for item in report.values(): + if isinstance(item, list): + item.sort() + return report, diff + + +# starting point +def comparer_run() -> Dict[str, Any]: + result = {} + for release, urls in RELEASES.items(): + logging.info('Processing release %i', release) + + logging.info('downloading rhel oval') + rhel_file = download_oval(urls['rhel_oval_url']) + logging.info('parsing rhel oval') + rhel_oval_dict = parse_oval(rhel_file) + + logging.info('downloading alma oval') + alma_oval_file = download_oval(urls['alma_oval_url']) + logging.info('parsing alma oval') + alma_oval_dict = parse_oval(alma_oval_file) + + logging.info('downloading alma errata') + alma_errata_file = download_errata(urls['alma_errata_url'], release) + logging.info('parsing alma errata') + alma_errata_dict = parse_errata(alma_errata_file) + + logging.info('comparing rhel and alma') + report_release, diff_release = compare( + rhel_oval_dict, alma_oval_dict, alma_errata_dict) + result[release] = {'report': report_release, + 'diff': diff_release, + 'rhel_oval_url': urls['rhel_oval_url'], + 'alma_oval_url': urls['alma_oval_url'], + 'alma_errata_url': urls['alma_errata_url']} + + result['report_generated'] = datetime.datetime.now().timestamp() * 1000 + result['sa_not_before'] = NOT_BEFORE.timestamp() * 1000 + + return result diff --git a/albs_oval_erratta_diff/config.py b/albs_oval_erratta_diff/config.py new file mode 100644 index 0000000..73bedcd --- /dev/null +++ b/albs_oval_erratta_diff/config.py @@ -0,0 +1,21 @@ +from pathlib import Path +import datetime + +RELEASES = { + 8: {'rhel_oval_url': 'https://www.redhat.com/security/data/oval/v2/RHEL8/rhel-8.oval.xml.bz2', + 'alma_oval_url': 'https://repo.almalinux.org/security/oval/org.almalinux.alsa-8.xml.bz2', + 'alma_errata_url': "https://errata.almalinux.org/8/errata.full.json", }, + 9: {'rhel_oval_url': 'https://www.redhat.com/security/data/oval/v2/RHEL9/rhel-9.oval.xml.bz2', + 'alma_oval_url': 'https://repo.almalinux.org/security/oval/org.almalinux.alsa-9.xml.bz2', + 'alma_errata_url': "https://errata.almalinux.org/9/errata.full.json", } +} +LOG_FILE = Path('logs/albs-oval-errata-diff.log') +DIFF_FILE = Path('results/diff.json') +DOWNLOAD_DIR = Path('/tmp') +# not checking anything before RHEL-9.0 release +NOT_BEFORE = datetime.datetime(2022, 5, 18) +UPDATE_INTERVAL_MINUTES = 30 +SERVER_PORT = 3001 +SERVER_IP = "127.0.0.1" +SA_EXCLUDE = [] +PACKAGES_EXCLUDE = ["dotnet-sdk-3.1-source-built-artifacts"] diff --git a/albs_oval_erratta_diff/package.py b/albs_oval_erratta_diff/package.py new file mode 100644 index 0000000..03ca6f5 --- /dev/null +++ b/albs_oval_erratta_diff/package.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class Package: + """ + Package represents RPM package exstracted from RHEL OVAL + """ + name: str + version: str + + def __str__(self): + return f"{self.name}-{self.version}" diff --git a/albs_oval_erratta_diff/sa.py b/albs_oval_erratta_diff/sa.py new file mode 100644 index 0000000..7e4dbfb --- /dev/null +++ b/albs_oval_erratta_diff/sa.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from typing import List + +from .package import Package + + +@dataclass +class SecurityAdvisory: + """ + SecurityAdvisory represents Security advisory deffition extracted + from oval or errata + """ + title: str + id: str + packages: List[Package] diff --git a/albs_oval_erratta_diff/start.py b/albs_oval_erratta_diff/start.py new file mode 100644 index 0000000..bab92b9 --- /dev/null +++ b/albs_oval_erratta_diff/start.py @@ -0,0 +1,98 @@ +""" +service compares rhel oval with alma ovals and errata ovals +results available via API Call +""" +from aiohttp import web +import copy +import logging +import threading +from time import sleep +import json + +from .config import LOG_FILE, DIFF_FILE, UPDATE_INTERVAL_MINUTES, SERVER_IP, SERVER_PORT +from .comparer import comparer_run + + +# This dict holds all current differentes +diffs = {} +diffs_lock = threading.Lock() + + +async def web_handler(request): + data = {} + try: + diffs_lock.acquire() + data = copy.deepcopy(diffs) + diffs_lock.release() + except Exception as e: + logging.critical("Unhandled exeption %s", e, exc_info=True) + return web.json_response(data=data) + + +def webserver_run(): + app = web.Application() + app.add_routes([web.get('/', web_handler)]) + web.run_app(app=app, host=SERVER_IP, port=SERVER_PORT) + + +def diff_checker(): + while True: + logging.info("Start comparing") + # generating new diff + try: + result = comparer_run() + except Exception as e: + logging.critical("Unhandled exeption %s", e, exc_info=True) + else: + logging.info("Finished comparing, updating diff dict") + diffs_lock.acquire() + global diffs + diffs = result + diffs_lock.release() + # dumping + logging.info("Saving results to disk") + try: + with open(DIFF_FILE, 'w', encoding='utf-8') as flw: + json.dump(result, flw, indent=4) + except Exception as e: + logging.critical("Unhandled exeption %s", e, exc_info=True) + logging.info("Done") + + logging.info("Finished comparing, go to sleep for %d minutes", + UPDATE_INTERVAL_MINUTES) + sleep(UPDATE_INTERVAL_MINUTES * 60) + + +def start(): + # making sure that directory exists + for p in [LOG_FILE, DIFF_FILE]: + if not p.parent.exists(): + p.parent.mkdir() + + logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(levelname)s %(funcName)s %(message)s', + handlers=[logging.FileHandler(LOG_FILE, mode='a'), + logging.StreamHandler()]) + + logging.info("Trying to load diff file from disk") + try: + with open(DIFF_FILE, 'r', encoding='utf-8') as flr: + loaded_data = json.load(flr) + diffs_lock.acquire() + diffs = loaded_data + diffs_lock.release() + except Exception as e: + logging.warning('cant load data from disk %s', e) + else: + logging.info('diff file was loaded') + + logging.info("Starting diff_checker in background") + thread = threading.Thread(target=diff_checker) + thread.daemon = True + thread.start() + logging.info("Starting webserver") + webserver_run() + + +if __name__ == "__main__": + start() From 7844a44a74173d944bd755821a160006e9ea9e0b Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 28 Dec 2022 17:57:00 +0100 Subject: [PATCH 02/11] added log rotation --- albs_oval_erratta_diff/start.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/albs_oval_erratta_diff/start.py b/albs_oval_erratta_diff/start.py index bab92b9..15c3e35 100644 --- a/albs_oval_erratta_diff/start.py +++ b/albs_oval_erratta_diff/start.py @@ -5,6 +5,7 @@ results available via API Call from aiohttp import web import copy import logging +from logging.handlers import RotatingFileHandler import threading from time import sleep import json @@ -72,7 +73,8 @@ def start(): logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(funcName)s %(message)s', handlers=[logging.FileHandler(LOG_FILE, mode='a'), - logging.StreamHandler()]) + logging.StreamHandler(), + RotatingFileHandler(LOG_FILE, maxBytes=10000, backupCount=3)]) logging.info("Trying to load diff file from disk") try: From 48750d137772910046ca5e1cc1716b922cb14c8e Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 12:17:21 +0100 Subject: [PATCH 03/11] - added config generation logic - fixed typo in package name --- .gitignore | 3 +- albs-oval-errata-diff.py | 4 - albs_oval_errata_diff.py | 7 ++ .../__init__.py | 0 .../comparer.py | 0 albs_oval_errata_diff/config.py | 105 ++++++++++++++++++ .../package.py | 0 .../sa.py | 0 .../start.py | 0 albs_oval_erratta_diff/config.py | 21 ---- config.default.yml | 68 ++++++++++++ 11 files changed, 182 insertions(+), 26 deletions(-) delete mode 100644 albs-oval-errata-diff.py create mode 100644 albs_oval_errata_diff.py rename {albs_oval_erratta_diff => albs_oval_errata_diff}/__init__.py (100%) rename {albs_oval_erratta_diff => albs_oval_errata_diff}/comparer.py (100%) create mode 100644 albs_oval_errata_diff/config.py rename {albs_oval_erratta_diff => albs_oval_errata_diff}/package.py (100%) rename {albs_oval_erratta_diff => albs_oval_errata_diff}/sa.py (100%) rename {albs_oval_erratta_diff => albs_oval_errata_diff}/start.py (100%) delete mode 100644 albs_oval_erratta_diff/config.py create mode 100644 config.default.yml diff --git a/.gitignore b/.gitignore index 27ace47..886279f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ venv logs results *.pyc -__pycache__ \ No newline at end of file +__pycache__ +.vscode \ No newline at end of file diff --git a/albs-oval-errata-diff.py b/albs-oval-errata-diff.py deleted file mode 100644 index 8c59f90..0000000 --- a/albs-oval-errata-diff.py +++ /dev/null @@ -1,4 +0,0 @@ -from albs_oval_erratta_diff.start import start - - -start() diff --git a/albs_oval_errata_diff.py b/albs_oval_errata_diff.py new file mode 100644 index 0000000..d878108 --- /dev/null +++ b/albs_oval_errata_diff.py @@ -0,0 +1,7 @@ +""" +albs_oval_errata_diff.py is a service startup script +""" +from albs_oval_errata_diff.start import start + + +start() diff --git a/albs_oval_erratta_diff/__init__.py b/albs_oval_errata_diff/__init__.py similarity index 100% rename from albs_oval_erratta_diff/__init__.py rename to albs_oval_errata_diff/__init__.py diff --git a/albs_oval_erratta_diff/comparer.py b/albs_oval_errata_diff/comparer.py similarity index 100% rename from albs_oval_erratta_diff/comparer.py rename to albs_oval_errata_diff/comparer.py diff --git a/albs_oval_errata_diff/config.py b/albs_oval_errata_diff/config.py new file mode 100644 index 0000000..ab69d67 --- /dev/null +++ b/albs_oval_errata_diff/config.py @@ -0,0 +1,105 @@ +''' +config.py used for generation service configuration based on input json file +''' + +from datetime import datetime, date +from pathlib import Path +from typing import Dict, List +from ipaddress import IPv4Address + +from pydantic import BaseModel, validator, Field # pylint: disable=import-error +import yaml + + +# DEFAULTS +DIFF_FILE = Path('/tmp/albs-oval-errata-diff.json') +DOWNLOAD_DIR = Path('/tmp') +LOG_FILE = Path('logs/albs-oval-errata-diff.log') +PACKAGES_EXCLUDE = [] +SA_EXCLUDE = [] +SERVER_PORT = 3001 +SERVER_IP = IPv4Address('127.0.0.1') +# not checking anything before RHEL-9.0 release +NOT_BEFORE = datetime(2022, 5, 18) +UPDATE_INTERVAL_MINUTES = 30 + + +class ReleaseUrls(BaseModel): + """ + ReleaseUrls represents list of RHEL/Alma Oval and Errata URLS for specific OS release + """ + rhel_oval_url: str = Field(description='URL for RHEL OVAL file') + alma_oval_url: str = Field(description='URL for Alma OVAL file') + alma_errata_url: str = Field(description='URL for Alma Errata file') + + +class Config(BaseModel): + """ + Config represents service configuration + """ + diff_file: Path = Field(description="file to store diff JSON in", + default=DIFF_FILE) + download_dir: Path = Field( + description='directory to download Oval/Errata files to', + default=DOWNLOAD_DIR) + log_file: Path = Field( + description='file to write logs to', + default=LOG_FILE) + packages_exclude: List[str] = Field( + description='list of RPM package names to exclude from checking', + default=PACKAGES_EXCLUDE) + releases: Dict[int, ReleaseUrls] = Field( + description='list of OS releases with Oval/Errata URLs to check') + sa_exclude: List[str] = Field( + description='list of Security Advisory IDs (ALSA-2022:5219) to exclude from checking', + default=SA_EXCLUDE) + server_port: int = Field( + description="port that will be used by websever", + default=SERVER_PORT) + server_ip: IPv4Address = Field( + description="IP that will be used by webserver", + default=SERVER_IP) + not_before: date = Field( + description='date to start checking from (YYYY-mm-dd)', + default=NOT_BEFORE) + update_interval_minutes: int = Field( + description='how often service will be running difference checks (in minutes)', + default=UPDATE_INTERVAL_MINUTES) + + @validator("releases", pre=True) + @classmethod + def parse_releases(cls, value) -> Dict[int, ReleaseUrls]: + """ + parse_release converts releases attribute + Dict[int, Dict[str, str]] -> Dict[str, ReleaseUrls] + """ + result: Dict[int, ReleaseUrls] = {} + for release, urls in value.items(): + result[release] = ReleaseUrls(rhel_oval_url=urls['rhel_oval_url'], + alma_oval_url=urls['alma_oval_url'], + alma_errata_url=urls['alma_errata_url']) + return result + + @validator("not_before", pre=True) + @classmethod + def str_to_datetime(cls, value) -> datetime: + """ + str_to_datetime converts string attr str -> datetime + """ + return datetime.strptime( + value, + "%Y-%m-%d" + ).date() + + +def get_config(yml_path: str) -> Config: + """ + get_config loads yml file and generates Config instance + """ + with open(yml_path, 'r', encoding='utf-8') as flr: + data = yaml.safe_load(flr) + return Config(**data) + + +if __name__ == "__main__": + print(get_config('./config.default.yml')) diff --git a/albs_oval_erratta_diff/package.py b/albs_oval_errata_diff/package.py similarity index 100% rename from albs_oval_erratta_diff/package.py rename to albs_oval_errata_diff/package.py diff --git a/albs_oval_erratta_diff/sa.py b/albs_oval_errata_diff/sa.py similarity index 100% rename from albs_oval_erratta_diff/sa.py rename to albs_oval_errata_diff/sa.py diff --git a/albs_oval_erratta_diff/start.py b/albs_oval_errata_diff/start.py similarity index 100% rename from albs_oval_erratta_diff/start.py rename to albs_oval_errata_diff/start.py diff --git a/albs_oval_erratta_diff/config.py b/albs_oval_erratta_diff/config.py deleted file mode 100644 index 73bedcd..0000000 --- a/albs_oval_erratta_diff/config.py +++ /dev/null @@ -1,21 +0,0 @@ -from pathlib import Path -import datetime - -RELEASES = { - 8: {'rhel_oval_url': 'https://www.redhat.com/security/data/oval/v2/RHEL8/rhel-8.oval.xml.bz2', - 'alma_oval_url': 'https://repo.almalinux.org/security/oval/org.almalinux.alsa-8.xml.bz2', - 'alma_errata_url': "https://errata.almalinux.org/8/errata.full.json", }, - 9: {'rhel_oval_url': 'https://www.redhat.com/security/data/oval/v2/RHEL9/rhel-9.oval.xml.bz2', - 'alma_oval_url': 'https://repo.almalinux.org/security/oval/org.almalinux.alsa-9.xml.bz2', - 'alma_errata_url': "https://errata.almalinux.org/9/errata.full.json", } -} -LOG_FILE = Path('logs/albs-oval-errata-diff.log') -DIFF_FILE = Path('results/diff.json') -DOWNLOAD_DIR = Path('/tmp') -# not checking anything before RHEL-9.0 release -NOT_BEFORE = datetime.datetime(2022, 5, 18) -UPDATE_INTERVAL_MINUTES = 30 -SERVER_PORT = 3001 -SERVER_IP = "127.0.0.1" -SA_EXCLUDE = [] -PACKAGES_EXCLUDE = ["dotnet-sdk-3.1-source-built-artifacts"] diff --git a/config.default.yml b/config.default.yml new file mode 100644 index 0000000..93c757a --- /dev/null +++ b/config.default.yml @@ -0,0 +1,68 @@ +--- +# diff_file +# file to store diff JSON in +# requred: no +# default: /tmp/albs-oval-errata-diff.json +diff_file: /tmp/albs-oval-errata-diff.json + +# download_dir +# directory to download Oval/Errata files to +# required: no +# default: /tmp +download_dir: /tmp + +# log_file +# file to write logs to +# requred: no +# default: logs/albs-oval-errata-diff.log +log_file: logs/albs-oval-errata-diff.log + +# packages_exclude +# list of RPM package names to exclude from checking +# requred: no +# default: [] +packages_exclude: [] + +# releases +# list of OS releases with Oval/Errata URLs to check +# required: yes +# default: N/A +releases: + 8: + rhel_oval_url: https://www.redhat.com/security/data/oval/v2/RHEL8/rhel-8.oval.xml.bz2 + alma_oval_url: https://repo.almalinux.org/security/oval/org.almalinux.alsa-8.xml.bz2 + alma_errata_url: https://errata.almalinux.org/8/errata.full.json + 9: + rhel_oval_url: https://www.redhat.com/security/data/oval/v2/RHEL9/rhel-9.oval.xml.bz2' + alma_oval_url: https://repo.almalinux.org/security/oval/org.almalinux.alsa-9.xml.bz2' + alma_errata_url: https://errata.almalinux.org/9/errata.full.json + +# sa_exclude +# list of Security Advisory IDs (ALSA-2022:5219) to exclude from checking +# requred: no +# default: [] +sa_exclude: [] + +# server_port +# port that will be used by websever +# required: no +# default: 3001 +server_port: 3001 + +# server_ip +# IP that will be used by webserver +# required: no +# default: 127.0.0.1 +server_ip: 127.0.0.1 + +# not_before +# date to start checking from (YYYY-mm-dd) +# required: no +# default: 2022-5-18 (Release of RHEL 9.0) +not_before: 2022-5-18 + +# update_interval_minutes +# how often service will be running difference checks (in minutes) +# required: no +# default: 30 +update_interval_minutes: 30 \ No newline at end of file From fb2b1de2c1d7ac677c0f326cfb94eebb9f020471 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 15:29:18 +0100 Subject: [PATCH 04/11] integrated config logic into a project --- albs_oval_errata_diff.py | 8 +- albs_oval_errata_diff/comparer.py | 123 +++++++++++++++++------------- albs_oval_errata_diff/config.py | 9 +-- albs_oval_errata_diff/package.py | 4 +- albs_oval_errata_diff/sa.py | 5 +- albs_oval_errata_diff/start.py | 93 +++++++++++++--------- config.default.yml | 4 +- 7 files changed, 146 insertions(+), 100 deletions(-) diff --git a/albs_oval_errata_diff.py b/albs_oval_errata_diff.py index d878108..206d00d 100644 --- a/albs_oval_errata_diff.py +++ b/albs_oval_errata_diff.py @@ -1,7 +1,11 @@ """ albs_oval_errata_diff.py is a service startup script """ +import sys from albs_oval_errata_diff.start import start - -start() +try: + YAML_PATH = sys.argv[1] +except IndexError: + print(f"Usage {sys.argv[0]} config.yml") +start(YAML_PATH) diff --git a/albs_oval_errata_diff/comparer.py b/albs_oval_errata_diff/comparer.py index 3d96971..18e7c29 100644 --- a/albs_oval_errata_diff/comparer.py +++ b/albs_oval_errata_diff/comparer.py @@ -1,80 +1,87 @@ +""" +package comparer.py implemets difference checking logic +""" + import bz2 import datetime +from pathlib import Path import re -import requests from typing import Tuple, List, Dict, Any -import xml.etree.ElementTree as ET import logging import json +import xml.etree.ElementTree as ET -from .config import DOWNLOAD_DIR, NOT_BEFORE, RELEASES, SA_EXCLUDE, PACKAGES_EXCLUDE -from .sa import SecurityAdvisory +import requests + +from .config import Config from .package import Package +from .sa import SecurityAdvisory -def download_oval(url: str) -> str: +def download_oval(url: str, download_dir: Path) -> str: """ download_oval downloads, decompreses oval file and returns filepath of saved file """ - r = requests.get(url, stream=True, timeout=30) + response = requests.get(url, stream=True, timeout=30) decompressor = bz2.BZ2Decompressor() fname = url.split('/')[-1].replace('.bz2', '') - fpath = DOWNLOAD_DIR / fname - with open(fpath, 'wb') as fd: - for chunk in r.iter_content(chunk_size=128): - fd.write(decompressor.decompress(chunk)) + fpath = download_dir / fname + with open(fpath, 'wb') as flw: + for chunk in response.iter_content(chunk_size=128): + flw.write(decompressor.decompress(chunk)) return fpath -def download_errata(url: str, release_version: int) -> str: +def download_errata(url: str, release_version: int, download_dir: Path) -> str: """ downloads errata_full.json file end returns file path """ response = requests.get(url, stream=True, timeout=30) fname = f'alma-{release_version}.json' - fpath = DOWNLOAD_DIR / fname + fpath = download_dir / fname with open(fpath, 'wb') as errata_file: for chunk in response.iter_content(chunk_size=128): errata_file.write(chunk) return fpath -def parse_oval(fpath: str) -> Dict[str, SecurityAdvisory]: +def parse_oval(fpath: str, not_before: datetime.datetime) -> Dict[str, SecurityAdvisory]: """ converting oval xml file to dict """ def extract_package(title: str) -> Package: - r = r'(.*) is earlier than \d+:(.+?(?=-))' - res = re.search(r, title) + regexp = r'(.*) is earlier than \d+:(.+?(?=-))' + res = re.search(regexp, title) name = res.group(1) version = res.group(2) return Package(name=name, version=version) def extract_id(title: str) -> str: - r = r'[RH|AL]SA-(\d{4}:\d+)(.*)' - res = re.search(r, title) + regexp = r'[RH|AL]SA-(\d{4}:\d+)(.*)' + res = re.search(regexp, title) return res.group(1) tree = ET.parse(fpath) root = tree.getroot() - ns = { + namespase = { 'n': 'http://oval.mitre.org/XMLSchema/oval-definitions-5', } res = {} - for definition in root.findall('n:definitions/', ns): - title = definition.find('n:metadata/n:title', ns).text + for definition in root.findall('n:definitions/', namespase): + title = definition.find('n:metadata/n:title', namespase).text issued = definition.find( - 'n:metadata/n:advisory/n:issued', ns).attrib['date'] + 'n:metadata/n:advisory/n:issued', namespase).attrib['date'] issued_dt = datetime.datetime.strptime(issued, "%Y-%m-%d") # we are only interesed in Security advisories after RHEL 8.3 - if ('RHSA' not in title and 'ALSA' not in title) or issued_dt < NOT_BEFORE: + if ('RHSA' not in title and 'ALSA' not in title) or issued_dt < not_before: continue sa_id = extract_id(title) - packages = [extract_package(i.attrib['comment']) for i in definition.findall(".//n:criterion", ns) + packages = [extract_package(i.attrib['comment']) for + i in definition.findall(".//n:criterion", namespase) if 'is earlier than' in i.attrib['comment']] res[sa_id] = SecurityAdvisory( title=title, id=sa_id, packages=packages) @@ -104,7 +111,9 @@ def parse_errata(fpath: str) -> Dict[str, SecurityAdvisory]: def compare(rhel_oval: Dict[str, SecurityAdvisory], alma_oval: Dict[str, SecurityAdvisory], - alma_errata: Dict[str, SecurityAdvisory]) -> Tuple[dict, list]: + alma_errata: Dict[str, SecurityAdvisory], + sa_exclude: List[str], + packages_exclude: List[str]) -> Tuple[dict, list]: """ compares rhel oval with alma oval and alma errata """ @@ -147,18 +156,18 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], sa_name = f'ALSA-{rhel_sa_id}' # filtering out SA - if sa_name in SA_EXCLUDE: + if sa_name in sa_exclude: report['excluded_sa'].append(sa_name) continue # filtefing out packages packages_to_check: List[Package] = [] - for p in rhel_sa.packages: - if any(p.name == i for i in PACKAGES_EXCLUDE): - if str(p) not in report['excluded_pkg']: - report['excluded_pkg'].append(str(p)) + for package in rhel_sa.packages: + if any(package.name == i for i in packages_exclude): + if str(package) not in report['excluded_pkg']: + report['excluded_pkg'].append(str(package)) else: - packages_to_check.append(p) + packages_to_check.append(package) # check oval try: @@ -175,13 +184,15 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], if r not in alma_oval_packages] if alma_oval_missing_packages: report['diff_count'] += 1 + diff_str = f"missing packages in oval SA: {','.join(alma_oval_missing_packages)}" diff.append({'sa_name': sa_name, - 'diff': f"missing packages in oval SA: {','.join(alma_oval_missing_packages)}"}) + 'diff': diff_str}) report['oval_missing_pkg_sa'].append(sa_name) report['oval_missing_pkg_sa_count'] += 1 - for mp in alma_oval_missing_packages: - if mp not in report['missing_packages_unique']: - report['missing_packages_unique'].append(mp) + for missing_package in alma_oval_missing_packages: + if missing_package not in report['missing_packages_unique']: + report['missing_packages_unique'].append( + missing_package) report['missing_packages_unique_count'] += 1 # check errata @@ -200,13 +211,14 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], str(r) for r in packages_to_check if r not in alma_errata_packages] if alma_errata_missing_packages: report['diff_count'] += 1 + diff_str = f"missing packages in errata SA: {','.join(alma_errata_missing_packages)}" diff.append({'sa_name': sa_name, - 'diff': f"missing packages in errata SA: {','.join(alma_errata_missing_packages)}"}) + 'diff': diff_str}) report['errata_missing_pkg_sa'].append(sa_name) report['errata_missing_pkg_sa_count'] += 1 - for mp in alma_errata_missing_packages: - if mp not in report['missing_packages_unique']: - report['missing_packages_unique'].append(mp) + for missing_package in alma_errata_missing_packages: + if missing_package not in report['missing_packages_unique']: + report['missing_packages_unique'].append(missing_package) report['missing_packages_unique_count'] += 1 else: # if we here, all checks were passed @@ -219,36 +231,45 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], # starting point -def comparer_run() -> Dict[str, Any]: +def comparer_run(config: Config) -> Dict[str, Any]: + """ + comperer_run is the starting point of comparer component + """ result = {} - for release, urls in RELEASES.items(): + for release, urls in config.releases.items(): logging.info('Processing release %i', release) logging.info('downloading rhel oval') - rhel_file = download_oval(urls['rhel_oval_url']) + rhel_file = download_oval(urls.rhel_oval_url, config.download_dir) logging.info('parsing rhel oval') - rhel_oval_dict = parse_oval(rhel_file) + rhel_oval_dict = parse_oval(rhel_file, config.not_before) logging.info('downloading alma oval') - alma_oval_file = download_oval(urls['alma_oval_url']) + alma_oval_file = download_oval( + urls.alma_oval_url, download_dir=config.download_dir) logging.info('parsing alma oval') - alma_oval_dict = parse_oval(alma_oval_file) + alma_oval_dict = parse_oval(alma_oval_file, config.not_before) logging.info('downloading alma errata') - alma_errata_file = download_errata(urls['alma_errata_url'], release) + alma_errata_file = download_errata(urls.alma_errata_url, + release, config.download_dir) logging.info('parsing alma errata') alma_errata_dict = parse_errata(alma_errata_file) logging.info('comparing rhel and alma') - report_release, diff_release = compare( - rhel_oval_dict, alma_oval_dict, alma_errata_dict) + report_release, diff_release = \ + compare(rhel_oval_dict, + alma_oval_dict, + alma_errata_dict, + config.sa_exclude, + config.packages_exclude) result[release] = {'report': report_release, 'diff': diff_release, - 'rhel_oval_url': urls['rhel_oval_url'], - 'alma_oval_url': urls['alma_oval_url'], - 'alma_errata_url': urls['alma_errata_url']} + 'rhel_oval_url': urls.rhel_oval_url, + 'alma_oval_url': urls.alma_oval_url, + 'alma_errata_url': urls.alma_errata_url} result['report_generated'] = datetime.datetime.now().timestamp() * 1000 - result['sa_not_before'] = NOT_BEFORE.timestamp() * 1000 + result['sa_not_before'] = config.not_before.timestamp() * 1000 return result diff --git a/albs_oval_errata_diff/config.py b/albs_oval_errata_diff/config.py index ab69d67..01c012d 100644 --- a/albs_oval_errata_diff/config.py +++ b/albs_oval_errata_diff/config.py @@ -2,12 +2,12 @@ config.py used for generation service configuration based on input json file ''' -from datetime import datetime, date +from datetime import datetime from pathlib import Path from typing import Dict, List from ipaddress import IPv4Address -from pydantic import BaseModel, validator, Field # pylint: disable=import-error +from pydantic import BaseModel, validator, Field # pylint: disable=import-error,no-name-in-module import yaml @@ -59,7 +59,7 @@ class Config(BaseModel): server_ip: IPv4Address = Field( description="IP that will be used by webserver", default=SERVER_IP) - not_before: date = Field( + not_before: datetime = Field( description='date to start checking from (YYYY-mm-dd)', default=NOT_BEFORE) update_interval_minutes: int = Field( @@ -88,8 +88,7 @@ class Config(BaseModel): """ return datetime.strptime( value, - "%Y-%m-%d" - ).date() + "%Y-%m-%d") def get_config(yml_path: str) -> Config: diff --git a/albs_oval_errata_diff/package.py b/albs_oval_errata_diff/package.py index 03ca6f5..3da1272 100644 --- a/albs_oval_errata_diff/package.py +++ b/albs_oval_errata_diff/package.py @@ -1,5 +1,7 @@ +""" +package.py contains Package dataclass definition +""" from dataclasses import dataclass -from typing import List @dataclass diff --git a/albs_oval_errata_diff/sa.py b/albs_oval_errata_diff/sa.py index 7e4dbfb..40e07da 100644 --- a/albs_oval_errata_diff/sa.py +++ b/albs_oval_errata_diff/sa.py @@ -1,3 +1,6 @@ +""" +sa contains SecurityAdvisory dataclass definition +""" from dataclasses import dataclass from typing import List @@ -11,5 +14,5 @@ class SecurityAdvisory: from oval or errata """ title: str - id: str + id: str # pylint: disable=invalid-name packages: List[Package] diff --git a/albs_oval_errata_diff/start.py b/albs_oval_errata_diff/start.py index 15c3e35..d2b0e7b 100644 --- a/albs_oval_errata_diff/start.py +++ b/albs_oval_errata_diff/start.py @@ -2,15 +2,17 @@ service compares rhel oval with alma ovals and errata ovals results available via API Call """ -from aiohttp import web import copy import logging from logging.handlers import RotatingFileHandler import threading from time import sleep +from ipaddress import IPv4Address import json -from .config import LOG_FILE, DIFF_FILE, UPDATE_INTERVAL_MINUTES, SERVER_IP, SERVER_PORT +from aiohttp import web + +from .config import get_config, Config from .comparer import comparer_run @@ -19,82 +21,97 @@ diffs = {} diffs_lock = threading.Lock() -async def web_handler(request): +async def web_handler(_): + """ + web_handler returns diffs as JSON file + """ data = {} try: diffs_lock.acquire() data = copy.deepcopy(diffs) diffs_lock.release() - except Exception as e: - logging.critical("Unhandled exeption %s", e, exc_info=True) + except Exception as err: # pylint: disable=broad-except + logging.critical("Unhandled exeption %s", err, exc_info=True) return web.json_response(data=data) -def webserver_run(): +def webserver_run(server_ip: IPv4Address, server_port: str): + """ + webserver_run starts webserver component + """ app = web.Application() app.add_routes([web.get('/', web_handler)]) - web.run_app(app=app, host=SERVER_IP, port=SERVER_PORT) + web.run_app(app=app, host=str(server_ip), port=server_port) -def diff_checker(): +def diff_checker(config: Config): + """ + runs comparer component in infinite loop + """ while True: logging.info("Start comparing") # generating new diff try: - result = comparer_run() - except Exception as e: - logging.critical("Unhandled exeption %s", e, exc_info=True) + result = comparer_run(config) + except Exception as err: # pylint: disable=broad-except + logging.critical("Unhandled exeption %s", err, exc_info=True) else: logging.info("Finished comparing, updating diff dict") diffs_lock.acquire() - global diffs + global diffs # pylint: disable=invalid-name,global-statement diffs = result diffs_lock.release() - # dumping - logging.info("Saving results to disk") - try: - with open(DIFF_FILE, 'w', encoding='utf-8') as flw: - json.dump(result, flw, indent=4) - except Exception as e: - logging.critical("Unhandled exeption %s", e, exc_info=True) + + # dumping + logging.info("Saving results to disk") + try: + with open(config.diff_file, 'w', encoding='utf-8') as flw: + json.dump(result, flw, indent=4) + except Exception as err: # pylint: disable=broad-except + logging.critical("Unhandled exeption %s", err, exc_info=True) logging.info("Done") logging.info("Finished comparing, go to sleep for %d minutes", - UPDATE_INTERVAL_MINUTES) - sleep(UPDATE_INTERVAL_MINUTES * 60) + config.update_interval_minutes) + sleep(config.update_interval_minutes * 60) -def start(): - # making sure that directory exists - for p in [LOG_FILE, DIFF_FILE]: - if not p.parent.exists(): - p.parent.mkdir() +def start(yaml_path: str): + """ + start starts comparer and webserver components + each component runs in it`s own thread + """ + config = get_config(yaml_path) + # making sure that parent directories exist + for path in [config.log_file, config.log_file]: + if not path.parent.exists(): + path.parent.mkdir() + + # configuring logging + handlers = [logging.FileHandler(config.log_file, mode='a'), + logging.StreamHandler(), + RotatingFileHandler(config.log_file, maxBytes=10000, backupCount=3)] logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(funcName)s %(message)s', - handlers=[logging.FileHandler(LOG_FILE, mode='a'), - logging.StreamHandler(), - RotatingFileHandler(LOG_FILE, maxBytes=10000, backupCount=3)]) + handlers=handlers) logging.info("Trying to load diff file from disk") try: - with open(DIFF_FILE, 'r', encoding='utf-8') as flr: + with open(config.diff_file, 'r', encoding='utf-8') as flr: loaded_data = json.load(flr) diffs_lock.acquire() + global diffs # pylint: disable=invalid-name,global-statement diffs = loaded_data diffs_lock.release() - except Exception as e: - logging.warning('cant load data from disk %s', e) + except Exception as err: # pylint: disable=broad-except + logging.warning('cant load data from disk %s', err) else: logging.info('diff file was loaded') logging.info("Starting diff_checker in background") - thread = threading.Thread(target=diff_checker) + thread = threading.Thread(target=diff_checker, args=(config,)) thread.daemon = True thread.start() logging.info("Starting webserver") - webserver_run() - - -if __name__ == "__main__": - start() + webserver_run(config.server_ip, config.server_port) diff --git a/config.default.yml b/config.default.yml index 93c757a..e597784 100644 --- a/config.default.yml +++ b/config.default.yml @@ -33,8 +33,8 @@ releases: alma_oval_url: https://repo.almalinux.org/security/oval/org.almalinux.alsa-8.xml.bz2 alma_errata_url: https://errata.almalinux.org/8/errata.full.json 9: - rhel_oval_url: https://www.redhat.com/security/data/oval/v2/RHEL9/rhel-9.oval.xml.bz2' - alma_oval_url: https://repo.almalinux.org/security/oval/org.almalinux.alsa-9.xml.bz2' + rhel_oval_url: https://www.redhat.com/security/data/oval/v2/RHEL9/rhel-9.oval.xml.bz2 + alma_oval_url: https://repo.almalinux.org/security/oval/org.almalinux.alsa-9.xml.bz2 alma_errata_url: https://errata.almalinux.org/9/errata.full.json # sa_exclude From b266b1b619ebc7c296c8913372ccd8ef10c0fa30 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 17:00:58 +0100 Subject: [PATCH 05/11] added Readme ans requirements.txt --- README.md | 63 ++++++++++++++++++++++++++++++++++ albs_oval_errata_diff/start.py | 6 ++-- requirements.txt | 4 +++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index e69de29..8474052 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,63 @@ +# albs-oval-errata-diff + +Service compares RHEL Oval with AlmaLinux Oval/Errata and stores differences. Differences are available via HTTP GET request in form of JSON + +## Components +### comparer +Downloads Oval/Errata files and generates differences report + +### webserver +Publishes JSON report via aiohttp webserver + +## Configuration +check [config.default.yml](config.default.yml) for references + +## Requirements +- Python 3.9 +- pip +- virtualenv + +## Installation +1. Checkout code + ```bash + $ git clone git@git.almalinux.org:kzhukov/albs-oval-errata-diff.git + ``` +2. Create and initialize virtual enviroment + ```bash + $ virtualenv -p python3.9 venv && source venv/bin/activate + ``` +3. Install requirements + ```bash + $ pip install -r requirements.txt + ``` +4. Create config file using [config.default.yml](config.default.yml) and start service with `albs_oval_errata_dif.py` script + ```bash + $ python albs_oval_errata_diff.py config.default.yml + 2022-12-29 16:20:11,139 INFO start Trying to load diff file from disk + 2022-12-29 16:20:11,142 INFO start Diff file was loaded + 2022-12-29 16:20:11,142 INFO start Starting diff_checker in background + 2022-12-29 16:20:11,143 INFO diff_checker Start comparing + 2022-12-29 16:20:11,143 INFO start Starting webserver + 2022-12-29 16:20:11,144 INFO comparer_run Processing release 8 + 2022-12-29 16:20:11,148 INFO comparer_run downloading rhel oval + ======== Running on http://127.0.0.1:3001 ======== + (Press CTRL+C to quit) + 2022-12-29 16:20:12,142 INFO comparer_run parsing rhel oval + 2022-12-29 16:20:13,154 INFO comparer_run downloading alma oval + 2022-12-29 16:20:16,516 INFO comparer_run parsing alma oval + 2022-12-29 16:20:17,695 INFO comparer_run downloading alma errata + 2022-12-29 16:20:28,894 INFO comparer_run parsing alma errata + 2022-12-29 16:20:29,143 INFO comparer_run comparing rhel and alma + 2022-12-29 16:20:29,233 INFO comparer_run Processing release 9 + 2022-12-29 16:20:29,234 INFO comparer_run downloading rhel oval + 2022-12-29 16:20:29,599 INFO comparer_run parsing rhel oval + 2022-12-29 16:20:29,716 INFO comparer_run downloading alma oval + 2022-12-29 16:20:31,033 INFO comparer_run parsing alma oval + 2022-12-29 16:20:31,165 INFO comparer_run downloading alma errata + 2022-12-29 16:20:33,542 INFO comparer_run parsing alma errata + 2022-12-29 16:20:33,601 INFO comparer_run comparing rhel and alma + 2022-12-29 16:20:33,621 INFO diff_checker Finished comparing, updating diff dict + 2022-12-29 16:20:33,622 INFO diff_checker Saving results to disk + 2022-12-29 16:20:33,630 INFO diff_checker Done + 2022-12-29 16:20:33,630 INFO diff_checker Finished comparing, go to sleep for 30 minutes + ``` \ No newline at end of file diff --git a/albs_oval_errata_diff/start.py b/albs_oval_errata_diff/start.py index d2b0e7b..bcccb23 100644 --- a/albs_oval_errata_diff/start.py +++ b/albs_oval_errata_diff/start.py @@ -46,7 +46,7 @@ def webserver_run(server_ip: IPv4Address, server_port: str): def diff_checker(config: Config): """ - runs comparer component in infinite loop + Runs comparer component in infinite loop """ while True: logging.info("Start comparing") @@ -105,9 +105,9 @@ def start(yaml_path: str): diffs = loaded_data diffs_lock.release() except Exception as err: # pylint: disable=broad-except - logging.warning('cant load data from disk %s', err) + logging.warning('Cant load data from disk %s', err) else: - logging.info('diff file was loaded') + logging.info('Diff file was loaded') logging.info("Starting diff_checker in background") thread = threading.Thread(target=diff_checker, args=(config,)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..37fd186 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +aiohttp==3.8.3 +pydantic==1.10.2 +PyYAML==6.0 +requests==2.28.1 From f4379c3f99fa6c3f65ae3d5201ee76b5946a91dd Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 17:08:51 +0100 Subject: [PATCH 06/11] Small fixes in log messages and Readme --- README.md | 4 ++-- albs_oval_errata_diff/comparer.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8474052..78375ec 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ check [config.default.yml](config.default.yml) for references ```bash $ pip install -r requirements.txt ``` -4. Create config file using [config.default.yml](config.default.yml) and start service with `albs_oval_errata_dif.py` script +4. Create config file using [config.default.yml](config.default.yml) and start service with _albs_oval_errata_dif.py_ script ```bash - $ python albs_oval_errata_diff.py config.default.yml + $ python albs_oval_errata_diff.py config.yml 2022-12-29 16:20:11,139 INFO start Trying to load diff file from disk 2022-12-29 16:20:11,142 INFO start Diff file was loaded 2022-12-29 16:20:11,142 INFO start Starting diff_checker in background diff --git a/albs_oval_errata_diff/comparer.py b/albs_oval_errata_diff/comparer.py index 18e7c29..4694707 100644 --- a/albs_oval_errata_diff/comparer.py +++ b/albs_oval_errata_diff/comparer.py @@ -239,24 +239,24 @@ def comparer_run(config: Config) -> Dict[str, Any]: for release, urls in config.releases.items(): logging.info('Processing release %i', release) - logging.info('downloading rhel oval') + logging.info('Downloading rhel oval') rhel_file = download_oval(urls.rhel_oval_url, config.download_dir) - logging.info('parsing rhel oval') + logging.info('Parsing rhel oval') rhel_oval_dict = parse_oval(rhel_file, config.not_before) - logging.info('downloading alma oval') + logging.info('Downloading alma oval') alma_oval_file = download_oval( urls.alma_oval_url, download_dir=config.download_dir) - logging.info('parsing alma oval') + logging.info('Parsing alma oval') alma_oval_dict = parse_oval(alma_oval_file, config.not_before) - logging.info('downloading alma errata') + logging.info('Downloading alma errata') alma_errata_file = download_errata(urls.alma_errata_url, release, config.download_dir) - logging.info('parsing alma errata') + logging.info('Parsing alma errata') alma_errata_dict = parse_errata(alma_errata_file) - logging.info('comparing rhel and alma') + logging.info('Comparing rhel and alma') report_release, diff_release = \ compare(rhel_oval_dict, alma_oval_dict, From 6a5f0471fefbdebcd0f361deabb0f3682a4c8ebc Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 17:10:00 +0100 Subject: [PATCH 07/11] typos --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78375ec..ac33a59 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ check [config.default.yml](config.default.yml) for references ```bash $ pip install -r requirements.txt ``` -4. Create config file using [config.default.yml](config.default.yml) and start service with _albs_oval_errata_dif.py_ script +4. Create config file using [config.default.yml](config.default.yml) and start service with _albs_oval_errata_diff.py_ script ```bash $ python albs_oval_errata_diff.py config.yml 2022-12-29 16:20:11,139 INFO start Trying to load diff file from disk From 3aee91a84faa5cd1fc5c948cf29d29b05993c8af Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 29 Dec 2022 17:17:48 +0100 Subject: [PATCH 08/11] config.py: changed " to ' --- albs_oval_errata_diff/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/albs_oval_errata_diff/config.py b/albs_oval_errata_diff/config.py index 01c012d..4896f88 100644 --- a/albs_oval_errata_diff/config.py +++ b/albs_oval_errata_diff/config.py @@ -54,10 +54,10 @@ class Config(BaseModel): description='list of Security Advisory IDs (ALSA-2022:5219) to exclude from checking', default=SA_EXCLUDE) server_port: int = Field( - description="port that will be used by websever", + description='port that will be used by websever', default=SERVER_PORT) server_ip: IPv4Address = Field( - description="IP that will be used by webserver", + description='IP that will be used by webserver', default=SERVER_IP) not_before: datetime = Field( description='date to start checking from (YYYY-mm-dd)', From 9a1045e638fe67bd88aa484b77bf82d7b00631ca Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 30 Dec 2022 09:50:32 +0100 Subject: [PATCH 09/11] added release.txt and commti tag --- releases.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 releases.txt diff --git a/releases.txt b/releases.txt new file mode 100644 index 0000000..59ea90e --- /dev/null +++ b/releases.txt @@ -0,0 +1,2 @@ +2022-12-30 v1.0.0 + First version of service From b1fc8b4281a2a2980b077449e9408a1d9719872c Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 30 Dec 2022 16:25:42 +0100 Subject: [PATCH 10/11] Removed unnecessary log handlers --- albs_oval_errata_diff.py | 3 ++- albs_oval_errata_diff/start.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/albs_oval_errata_diff.py b/albs_oval_errata_diff.py index 206d00d..c70fd0c 100644 --- a/albs_oval_errata_diff.py +++ b/albs_oval_errata_diff.py @@ -7,5 +7,6 @@ from albs_oval_errata_diff.start import start try: YAML_PATH = sys.argv[1] except IndexError: - print(f"Usage {sys.argv[0]} config.yml") + print(f"Usage: {sys.argv[0]} config.yml") + sys.exit(1) start(YAML_PATH) diff --git a/albs_oval_errata_diff/start.py b/albs_oval_errata_diff/start.py index bcccb23..8b681c8 100644 --- a/albs_oval_errata_diff/start.py +++ b/albs_oval_errata_diff/start.py @@ -89,12 +89,9 @@ def start(yaml_path: str): path.parent.mkdir() # configuring logging - handlers = [logging.FileHandler(config.log_file, mode='a'), - logging.StreamHandler(), - RotatingFileHandler(config.log_file, maxBytes=10000, backupCount=3)] logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(funcName)s %(message)s', - handlers=handlers) + handlers=[RotatingFileHandler(config.log_file, maxBytes=10000000, backupCount=3)]) logging.info("Trying to load diff file from disk") try: From 8bf501c9227532f5e19a5d6c580622f8eb92be4d Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 4 Jan 2023 11:15:29 +0100 Subject: [PATCH 11/11] ALBS-901 bugfix: false positive of SA with missing packages --- albs_oval_errata_diff/comparer.py | 7 ++++--- releases.txt | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/albs_oval_errata_diff/comparer.py b/albs_oval_errata_diff/comparer.py index 4694707..6c0c437 100644 --- a/albs_oval_errata_diff/comparer.py +++ b/albs_oval_errata_diff/comparer.py @@ -181,7 +181,7 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], # check if some packages are missing from oval SA alma_oval_packages = alma_oval_sa.packages alma_oval_missing_packages = [str(r) for r in packages_to_check - if r not in alma_oval_packages] + if str(r) not in [str(i) for i in alma_oval_packages]] if alma_oval_missing_packages: report['diff_count'] += 1 diff_str = f"missing packages in oval SA: {','.join(alma_oval_missing_packages)}" @@ -207,8 +207,9 @@ def compare(rhel_oval: Dict[str, SecurityAdvisory], continue # check if some packages are missing from errata SA alma_errata_packages = alma_errata_sa.packages - alma_errata_missing_packages = [ - str(r) for r in packages_to_check if r not in alma_errata_packages] + alma_errata_missing_packages = \ + [str(r) for r in packages_to_check + if str(r) not in [str(i) for i in alma_errata_packages]] if alma_errata_missing_packages: report['diff_count'] += 1 diff_str = f"missing packages in errata SA: {','.join(alma_errata_missing_packages)}" diff --git a/releases.txt b/releases.txt index 59ea90e..94b3d83 100644 --- a/releases.txt +++ b/releases.txt @@ -1,2 +1,4 @@ 2022-12-30 v1.0.0 First version of service +2023-01-04 v1.0.1 + Fixed missing packages false positives \ No newline at end of file