# pylint: disable=relative-beyond-top-level import logging from typing import List, Dict from ..models.extractor_config import ExtractorConfig from ..models.enums import BuildTaskEnum from ..models.build import BuildTask from ..db import DB from ..api_client import APIclient class Extractor: def __init__(self, config: ExtractorConfig, api: APIclient, db: DB): self.oldest_build_age = config.oldest_build_age self.api = api self.db = db def extract_and_store(self) -> int: build_count = 0 page_num = 1 last_build_id = self.db.get_latest_build_id() if not last_build_id: last_build_id = 0 logging.info("last_build_id: %s", last_build_id) stop = False while not stop: logging.info("page: %s", page_num) for build in self.api.get_builds(page_num): # check if we shoud stop processing build if build.id <= last_build_id or \ build.created_at <= self.oldest_build_age: stop = True break # inserting build and build tasks logging.info("inserting %s", build.id) self.db.insert_build(build.as_db_model()) for build_task in build.build_tasks: self.db.insert_buildtask(build_task.as_db_model()) build_count += 1 page_num += 1 return build_count def build_cleanup(self): logging.info('Removing all buidls older then %s', self.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S")) removed_count = self.db.cleanup_builds(self.oldest_build_age) logging.info('removed %d entries', removed_count) def __update_build_tasks_statuses(self, build_tasks: List[BuildTask], build_tasks_status_db: Dict[int, int]): for b in build_tasks: if b.status_id != build_tasks_status_db[b.id]: logging.info('build taks %d status have changed %s -> %s. Updating DB', b.id, BuildTaskEnum( build_tasks_status_db[b.id]).name, BuildTaskEnum(b.status_id).name) try: self.db.update_build_task(b.as_db_model()) except Exception as err: # pylint: disable=broad-except logging.error( 'failed to update build task %d: %s', b.id, err, exc_info=True) else: logging.info('build task %d was updated', b.id) else: logging.info( "build_task %d is still %s. Skipping", b.id, BuildTaskEnum(b.status_id).name) def update_builds(self): logging.info('Getting list of tasks from DB') unfinished_tasks = self.db.get_unfinished_builds() for build_id, build_tasks_db in unfinished_tasks.items(): try: logging.info('Getting status of build %d', build_id) build = self.api.get_build(build_id) logging.info('Updating build tasks') build_tasks_to_check = [ b for b in build.build_tasks if b.id in build_tasks_db] self.__update_build_tasks_statuses( build_tasks_to_check, build_tasks_db) if build.finished_at: logging.info( "build is finished, we need to update finished_at attribute") self.db.update_build(build.as_db_model()) logging.info('finished proccessing build %d', build_id) except Exception as err: # pylint: disable=broad-except logging.error("Cant process build %d: %s", build_id, err, exc_info=True)