91 lines
3.1 KiB
Python
91 lines
3.1 KiB
Python
from datetime import datetime, timedelta
|
|
import logging
|
|
from logging.handlers import RotatingFileHandler
|
|
import time
|
|
|
|
import yaml
|
|
|
|
# pylint: disable=relative-beyond-top-level
|
|
from ..api_client import APIclient
|
|
from ..db import DB
|
|
from .extractor import Extractor
|
|
from ..models.extractor_config import ExtractorConfig
|
|
from ..models.db_config import DbConfig
|
|
|
|
|
|
def __get_oldest_build_age(config: dict) -> datetime:
|
|
oldest_build_age = datetime.now().astimezone() \
|
|
- timedelta(days=config['data_store_days'])
|
|
return oldest_build_age
|
|
|
|
|
|
def __get_db_config(config: dict) -> DbConfig:
|
|
return DbConfig(name=config['db_name'],
|
|
port=int(config['db_port']),
|
|
host=config['db_host'],
|
|
username=config['db_username'],
|
|
password=config['db_password'])
|
|
|
|
|
|
def __get_config(yml_path: str) -> ExtractorConfig:
|
|
"""
|
|
get_config loads yml file and generates instance
|
|
"""
|
|
with open(yml_path, 'r', encoding='utf-8') as flr:
|
|
raw = yaml.safe_load(flr)
|
|
|
|
# adding new attrs
|
|
raw['oldest_build_age'] = __get_oldest_build_age(raw)
|
|
raw['db_config'] = __get_db_config(raw)
|
|
|
|
return ExtractorConfig(**raw)
|
|
|
|
|
|
def start(yml_path: str):
|
|
config = __get_config(yml_path)
|
|
|
|
# configuring logging
|
|
logging.basicConfig(level=logging.INFO,
|
|
format='%(asctime)s %(levelname)s <%(funcName)s> %(message)s',
|
|
handlers=[RotatingFileHandler(config.log_file,
|
|
maxBytes=10000000,
|
|
backupCount=3)])
|
|
|
|
while True:
|
|
logging.info('Starting extraction proccess')
|
|
api = APIclient(api_root=config.albs_url,
|
|
jwt=config.jwt, timeout=config.api_timeout)
|
|
db = DB(config.db_config)
|
|
extractor = Extractor(config, api, db)
|
|
|
|
logging.info('Starting builds insertion')
|
|
inserted_count = -1
|
|
try:
|
|
inserted_count = extractor.extract_and_store()
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.critical("Unhandled exception %s", err, exc_info=True)
|
|
else:
|
|
logging.info(
|
|
'Build extaction was finished. %d builds were inserted', inserted_count)
|
|
|
|
logging.info('Starting old builds removal')
|
|
try:
|
|
extractor.build_cleanup()
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.critical("Unhandled exception %s", err, exc_info=True)
|
|
else:
|
|
logging.info('Cleanup finished')
|
|
|
|
logging.info('Updating statuses of unfinished build tasks')
|
|
try:
|
|
extractor.update_builds()
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.critical("Unhandled exception %s", err, exc_info=True)
|
|
else:
|
|
logging.info('Update finished')
|
|
|
|
extractor.db.close_conn()
|
|
logging.info("Extraction was finished")
|
|
logging.info("Sleeping for %d seconds", config.scrape_inteval)
|
|
time.sleep(config.scrape_inteval)
|