# pylint: disable=relative-beyond-top-level import logging from typing import Dict, List from ..api_client import APIclient from ..const import BuildTaskEnum from ..db import DB from ..models.build import BuildTask from ..models.extractor_config import ExtractorConfig class Extractor: def __init__(self, config: ExtractorConfig, api: APIclient, db: DB): self.config = config self.api = api self.db = db def extract_and_store(self) -> int: build_count = 0 page_num = 1 last_build_id = self.db.get_latest_build_id() if not last_build_id: last_build_id = self.config.start_from - 1 logging.info("last_build_id: %s", last_build_id) stop = False while not stop: logging.info("page: %s", page_num) for build in self.api.get_builds(page_num): # check if we shoud stop processing build if build.id <= last_build_id or \ build.created_at <= self.config.oldest_build_age: stop = True break # some builds could move from one page to another if self.db.row_exists(pk=build.id, table='builds'): continue # inserting build build tasks and build tasks statistics logging.info('inserting %s', build.id) try: self.db.insert_build(build.as_db_model()) except Exception as error: # pylint: disable=broad-except logging.error('failed to insert build %d: %s', build.id, error, exc_info=True) continue for build_task in build.build_tasks: logging.info('build %s: inserting build task %s', build.id, build_task.id) try: self.db.insert_buildtask(build_task.as_db_model(), build_task.web_node_stats.as_db_model( build_task.id), build_task.build_node_stats.as_db_model( build_task.id)) except Exception as error: # pylint: disable=broad-except logging.error('build %s: failed to insert build task %d: %s', build.id, build_task.id, error, exc_info=True) logging.info( 'getting test tasks for build task %s', build_task.id) test_tasks = self.api.get_test_tasks(build_task.id) logging.info('received %d tests tasks', len(test_tasks)) if len(test_tasks) > 0: logging.info('inserting test tasks') as_db = [t.as_db_model() for t in test_tasks] self.db.insert_update_test_tasks(as_db) build_count += 1 page_num += 1 return build_count def build_cleanup(self): logging.info('Removing all buidls older then %s', self.config.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S")) removed_count = self.db.cleanup_builds(self.config.oldest_build_age) logging.info('removed %d entries', removed_count) def __update_build_tasks(self, build_tasks: List[BuildTask], build_tasks_status_db: Dict[int, int]): for b in build_tasks: if b.status_id != build_tasks_status_db[b.id]: logging.info('build: %s, build task %d status have changed %s -> %s. Updating DB', b.build_id, b.id, BuildTaskEnum( build_tasks_status_db[b.id]).name, BuildTaskEnum(b.status_id).name) try: self.db.update_build_task(b.as_db_model(), b.web_node_stats.as_db_model( b.id), b.build_node_stats.as_db_model(b.id)) except Exception as err: # pylint: disable=broad-except logging.error( 'build: %d, failed to update build task %d: %s', b.build_id, b.id, err, exc_info=True) else: logging.info( 'build: %d, build task %d was updated', b.build_id, b.id) else: logging.info( "build: %d, build_task %d is still %s. Skipping", b.build_id, b.id, BuildTaskEnum(b.status_id).name) def update_builds(self): logging.info('Getting list of tasks from DB') unfinished_tasks = self.db.get_unfinished_builds( self.config.oldest_to_update) for build_id, build_tasks_db in unfinished_tasks.items(): try: logging.info('Getting status of build %d', build_id) build = self.api.get_build(build_id) logging.info('Updating build tasks') build_tasks_to_check = [ b for b in build.build_tasks if b.id in build_tasks_db] self.__update_build_tasks( build_tasks_to_check, build_tasks_db) if build.finished_at: logging.info( "build is finished, we need to update finished_at attribute") self.db.update_build(build.as_db_model()) logging.info('finished proccessing build %d', build_id) except Exception as err: # pylint: disable=broad-except logging.error("Cant process build %d: %s", build_id, err, exc_info=True) def updating_test_tasks(self): logging.info('getting build task ids of unfinished tests') build_task_ids = self.db.get_build_tasks_for_unfinished_tests( self.config.oldest_to_update) for build_task_id in build_task_ids: try: logging.info('getting tests for build task %s', build_task_id) tasks_api = self.api.get_test_tasks(build_task_id) logging.info('updating test tasks') tasks_db = [t.as_db_model() for t in tasks_api] self.db.insert_update_test_tasks(tasks_db) except Exception as err: # pylint: disable=broad-except logging.error( 'failed to update tests for %d build task: %s', build_task_id, err, exc_info=True)