148 lines
6.7 KiB
Python
148 lines
6.7 KiB
Python
# pylint: disable=relative-beyond-top-level
|
|
|
|
import logging
|
|
from typing import Dict, List
|
|
|
|
from ..api_client import APIclient
|
|
from ..const import BuildTaskEnum
|
|
from ..db import DB
|
|
from ..models.build import BuildTask
|
|
from ..models.extractor_config import ExtractorConfig
|
|
|
|
|
|
class Extractor:
|
|
def __init__(self, config: ExtractorConfig, api: APIclient, db: DB):
|
|
self.config = config
|
|
self.api = api
|
|
self.db = db
|
|
|
|
def extract_and_store(self) -> int:
|
|
build_count = 0
|
|
page_num = 1
|
|
last_build_id = self.db.get_latest_build_id()
|
|
if not last_build_id:
|
|
last_build_id = self.config.start_from - 1
|
|
logging.info("last_build_id: %s", last_build_id)
|
|
stop = False
|
|
|
|
while not stop:
|
|
logging.info("page: %s", page_num)
|
|
for build in self.api.get_builds(page_num):
|
|
# check if we shoud stop processing build
|
|
if build.id <= last_build_id or \
|
|
build.created_at <= self.config.oldest_build_age:
|
|
stop = True
|
|
break
|
|
|
|
# some builds could move from one page to another
|
|
if self.db.row_exists(pk=build.id, table='builds'):
|
|
continue
|
|
|
|
# inserting build build tasks and build tasks statistics
|
|
logging.info('inserting %s', build.id)
|
|
try:
|
|
self.db.insert_build(build.as_db_model())
|
|
except Exception as error: # pylint: disable=broad-except
|
|
logging.error('failed to insert build %d: %s',
|
|
build.id, error, exc_info=True)
|
|
continue
|
|
|
|
for build_task in build.build_tasks:
|
|
logging.info('build %s: inserting build task %s',
|
|
build.id, build_task.id)
|
|
try:
|
|
self.db.insert_buildtask(build_task.as_db_model(),
|
|
build_task.web_node_stats.as_db_model(
|
|
build_task.id),
|
|
build_task.build_node_stats.as_db_model(
|
|
build_task.id))
|
|
except Exception as error: # pylint: disable=broad-except
|
|
logging.error('build %s: failed to insert build task %d: %s',
|
|
build.id, build_task.id, error, exc_info=True)
|
|
|
|
logging.info(
|
|
'getting test tasks for build task %s', build_task.id)
|
|
test_tasks = self.api.get_test_tasks(build_task.id)
|
|
logging.info('received %d tests tasks', len(test_tasks))
|
|
if len(test_tasks) > 0:
|
|
logging.info('inserting test tasks')
|
|
as_db = [t.as_db_model() for t in test_tasks]
|
|
self.db.insert_update_test_tasks(as_db)
|
|
build_count += 1
|
|
page_num += 1
|
|
return build_count
|
|
|
|
def build_cleanup(self):
|
|
logging.info('Removing all buidls older then %s',
|
|
self.config.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
|
|
removed_count = self.db.cleanup_builds(self.config.oldest_build_age)
|
|
logging.info('removed %d entries', removed_count)
|
|
|
|
def __update_build_tasks(self, build_tasks: List[BuildTask],
|
|
build_tasks_status_db: Dict[int, int]):
|
|
for b in build_tasks:
|
|
if b.status_id != build_tasks_status_db[b.id]:
|
|
logging.info('build: %s, build task %d status have changed %s -> %s. Updating DB',
|
|
b.build_id,
|
|
b.id, BuildTaskEnum(
|
|
build_tasks_status_db[b.id]).name,
|
|
BuildTaskEnum(b.status_id).name)
|
|
try:
|
|
self.db.update_build_task(b.as_db_model(),
|
|
b.web_node_stats.as_db_model(
|
|
b.id),
|
|
b.build_node_stats.as_db_model(b.id))
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.error(
|
|
'build: %d, failed to update build task %d: %s',
|
|
b.build_id, b.id, err, exc_info=True)
|
|
else:
|
|
logging.info(
|
|
'build: %d, build task %d was updated', b.build_id, b.id)
|
|
else:
|
|
logging.info(
|
|
"build: %d, build_task %d is still %s. Skipping",
|
|
b.build_id, b.id, BuildTaskEnum(b.status_id).name)
|
|
|
|
def update_builds(self):
|
|
logging.info('Getting list of tasks from DB')
|
|
unfinished_tasks = self.db.get_unfinished_builds(
|
|
self.config.oldest_to_update)
|
|
for build_id, build_tasks_db in unfinished_tasks.items():
|
|
try:
|
|
logging.info('Getting status of build %d', build_id)
|
|
build = self.api.get_build(build_id)
|
|
|
|
logging.info('Updating build tasks')
|
|
build_tasks_to_check = [
|
|
b for b in build.build_tasks if b.id in build_tasks_db]
|
|
self.__update_build_tasks(
|
|
build_tasks_to_check, build_tasks_db)
|
|
|
|
if build.finished_at:
|
|
logging.info(
|
|
"build is finished, we need to update finished_at attribute")
|
|
self.db.update_build(build.as_db_model())
|
|
|
|
logging.info('finished proccessing build %d', build_id)
|
|
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.error("Cant process build %d: %s",
|
|
build_id, err, exc_info=True)
|
|
|
|
def updating_test_tasks(self):
|
|
logging.info('getting build task ids of unfinished tests')
|
|
build_task_ids = self.db.get_build_tasks_for_unfinished_tests(
|
|
self.config.oldest_to_update)
|
|
for build_task_id in build_task_ids:
|
|
try:
|
|
logging.info('getting tests for build task %s', build_task_id)
|
|
tasks_api = self.api.get_test_tasks(build_task_id)
|
|
logging.info('updating test tasks')
|
|
tasks_db = [t.as_db_model() for t in tasks_api]
|
|
self.db.insert_update_test_tasks(tasks_db)
|
|
except Exception as err: # pylint: disable=broad-except
|
|
logging.error(
|
|
'failed to update tests for %d build task: %s',
|
|
build_task_id, err, exc_info=True)
|