from datetime import datetime, timedelta import logging from logging.handlers import RotatingFileHandler import yaml from ..api_client import APIclient from ..db import DB from .extractor import Extractor from ..models.extractor_config import ExtractorConfig from ..models.db_config import DbConfig def __get_oldest_build_age(config: dict) -> datetime: oldest_build_age = datetime.now().astimezone() \ - timedelta(days=config['data_store_days']) return oldest_build_age def __get_db_config(config: dict) -> DbConfig: return DbConfig(name=config['db_name'], port=int(config['db_port']), host=config['db_host'], username=config['db_username'], password=config['db_password']) def __get_config(yml_path: str) -> ExtractorConfig: """ get_config loads yml file and generates instance """ with open(yml_path, 'r', encoding='utf-8') as flr: raw = yaml.safe_load(flr) # adding new attrs raw['oldest_build_age'] = __get_oldest_build_age(raw) raw['db_config'] = __get_db_config(raw) return ExtractorConfig(**raw) def start(yml_path: str): config = __get_config(yml_path) # configuring logging logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s <%(funcName)s> %(message)s', handlers=[RotatingFileHandler(config.log_file, maxBytes=10000000, backupCount=3)]) api = APIclient(api_root=config.albs_url, jwt=config.jwt) db = DB(config.db_config) extractor = Extractor(config, api, db) logging.info('Starting builds insertion') inserted_count = -1 try: inserted_count = extractor.extract_and_store() except Exception as err: # pylint: disable=broad-except logging.critical("Unhandled exeption %s", err, exc_info=True) else: logging.info( 'Build extaction was finished. %d builds were inserted', inserted_count) logging.info('Starting old builds removal') try: extractor.build_cleanup() except Exception as err: logging.critical("Unhandled exeption %s", err, exc_info=True) else: logging.info('Cleanup finished')