46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
|
import logging
|
||
|
|
|
||
|
|
from ..models.extractor_config import ExtractorConfig
|
||
|
|
from ..api_client import APIclient
|
||
|
|
from ..db import DB
|
||
|
|
|
||
|
|
|
||
|
|
class Extractor:
|
||
|
|
def __init__(self, config: ExtractorConfig, api: APIclient, db: DB):
|
||
|
|
self.oldest_build_age = config.oldest_build_age
|
||
|
|
self.api = api
|
||
|
|
self.db = db
|
||
|
|
|
||
|
|
def extract_and_store(self) -> int:
|
||
|
|
build_count = 0
|
||
|
|
page_num = 1
|
||
|
|
last_build_id = self.db.get_latest_build_id()
|
||
|
|
if not last_build_id:
|
||
|
|
last_build_id = 0
|
||
|
|
logging.info(f"last_build_id: {last_build_id}")
|
||
|
|
stop = False
|
||
|
|
|
||
|
|
while not stop:
|
||
|
|
logging.info(f"page: {page_num}")
|
||
|
|
for build in self.api.get_builds(page_num):
|
||
|
|
# check if we shoud stop processing build
|
||
|
|
if build.id <= last_build_id or \
|
||
|
|
build.created_at <= self.oldest_build_age:
|
||
|
|
stop = True
|
||
|
|
break
|
||
|
|
|
||
|
|
# inserting build and build tasks
|
||
|
|
logging.info(f"inserting {build.id}")
|
||
|
|
self.db.insert_update_build(build.as_db_model())
|
||
|
|
for build_task in build.build_tasks:
|
||
|
|
self.db.insert_update_buildtask(build_task.as_db_model())
|
||
|
|
build_count += 1
|
||
|
|
page_num += 1
|
||
|
|
return build_count
|
||
|
|
|
||
|
|
def build_cleanup(self):
|
||
|
|
logging.info('Removing all buidls older then %s',
|
||
|
|
self.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
|
||
|
|
removed_count = self.db.cleanup_builds(self.oldest_build_age)
|
||
|
|
logging.info('removed %d entries', removed_count)
|