kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

db: debug of update feature

Browse Source
This commit is contained in:
Kirill Zhukov 2023-03-16 22:48:35 +01:00
parent 679328093a
commit 313d4a4d2a
2 changed files with 30 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
build_analytics/build_analytics/db.py
View File

@ -10,7 +10,6 @@ from .models.build_node_stat_db import BuildNodeStatDB
from .models.db_config import DbConfig from .models.db_config import DbConfig
from .models.web_node_stat_db import WebNodeStatDB from .models.web_node_stat_db import WebNodeStatDB
from .models.test_task_db import TestTaskDB from .models.test_task_db import TestTaskDB
from .models.test_step_stat_db import TestStepStatDB
class DB(): class DB():
@ -109,9 +108,10 @@ class DB():
res: Dict[int, Dict[int, int]] = {} res: Dict[int, Dict[int, int]] = {}
# getting unfinished builds # getting unfinished builds
sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s ;' sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
cur = self.__conn.cursor(not_before.timestamp()) cur = self.__conn.cursor()
cur.execute(sql) cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
for row in cur.fetchall(): for row in cur.fetchall():
res[row[0]] = {} res[row[0]] = {}
@ -163,7 +163,8 @@ class DB():
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id) 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(task_id=stat.build_task_id, if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id, stat_name_id=stat.stat_name_id,
table_name='web_node_stats'): table_name='web_node_stats',
column_name='build_task_id'):
sql = ''' sql = '''
UPDATE web_node_stats UPDATE web_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -187,7 +188,8 @@ class DB():
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id) 'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(task_id=stat.build_task_id, if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id, stat_name_id=stat.stat_name_id,
table_name='build_node_stats'): table_name='build_node_stats',
column_name='build_task_id'):
sql = ''' sql = '''
UPDATE build_node_stats UPDATE build_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -221,11 +223,11 @@ class DB():
return None return None
return int(val[0]) return int(val[0])
def stat_exists(self, task_id: int, stat_name_id: int, table_name: str) -> bool: def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool:
sql = f''' sql = f'''
SELECT COUNT(build_task_id) SELECT COUNT({column_name})
FROM {table_name} FROM {table_name}
WHERE build_task_id = %s AND stat_name_id = %s; WHERE {column_name} = %s AND stat_name_id = %s;
''' '''
cur = self.__conn.cursor() cur = self.__conn.cursor()
cur.execute(sql, (task_id, stat_name_id)) cur.execute(sql, (task_id, stat_name_id))
@ -263,13 +265,14 @@ class DB():
''' '''
cur = self.__conn.cursor() cur = self.__conn.cursor()
sql = ''' sql = '''
SELECT bt.id SELECT DISTINCT bt.id
FROM build_tasks as bt FROM build_tasks as bt
INNER JOIN test_tasks AS tt INNER JOIN test_tasks AS tt
ON bt.id = tt.build_task_id ON bt.id = tt.build_task_id
WHERE tt.status_id < 3 AND tt.started_at > %s; WHERE tt.status_id < 3 AND bt.started_at > %s;
''' '''
cur.execute(sql, (not_before,)) cur.execute(sql, (not_before.timestamp(),))
logging.info('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()] result = [int(row[0]) for row in cur.fetchall()]
return result return result
@ -286,18 +289,27 @@ class DB():
''' '''
cur.execute(sql, (task.revision, task.status_id, cur.execute(sql, (task.revision, task.status_id,
task.started_at, task.id)) task.started_at, task.id))
assert cur.rowcount == 1
# test step # test step
if not task.steps_stats: if not task.steps_stats:
continue continue
for s in task.steps_stats: for s in task.steps_stats:
if self.stat_exists(s.test_task_id, s.stat_name_id, 'test_steps_stats'): logging.info('test_task_id %s, stat_name_id %s',
s.test_task_id, s.stat_name_id)
if self.stat_exists(s.test_task_id,
s.stat_name_id,
'test_steps_stats',
'test_task_id'):
sql = ''' sql = '''
UPDATE test_steps_stats UPDATE test_steps_stats
SET start_ts = %s, SET start_ts = %s,
SET finish_ts = %s; finish_ts = %s
WHERE test_task_id = %s AND stat_name_id = %s;
''' '''
cur.execute(sql, s.start_ts, s.finish_ts) cur.execute(sql, (s.start_ts, s.finish_ts,
s.test_task_id, s.stat_name_id))
assert cur.rowcount == 1
else: else:
sql = ''' sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts) INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)

5
build_analytics/build_analytics/extractor/extractor.py
View File

@ -102,7 +102,8 @@ class Extractor:
def update_builds(self): def update_builds(self):
logging.info('Getting list of tasks from DB') logging.info('Getting list of tasks from DB')
unfinished_tasks = self.db.get_unfinished_builds() unfinished_tasks = self.db.get_unfinished_builds(
self.config.oldest_to_update)
for build_id, build_tasks_db in unfinished_tasks.items(): for build_id, build_tasks_db in unfinished_tasks.items():
try: try:
logging.info('Getting status of build %d', build_id) logging.info('Getting status of build %d', build_id)
@ -136,7 +137,7 @@ class Extractor:
logging.info('updating test tasks') logging.info('updating test tasks')
tasks_db = [t.as_db_model() for t in tasks_api] tasks_db = [t.as_db_model() for t in tasks_api]
self.db.update_test_tasks(tasks_db) self.db.update_test_tasks(tasks_db)
except Exception as err: except Exception as err: # pylint: disable=broad-except
logging.error( logging.error(
'failed to update tests for %d build task: %s', 'failed to update tests for %d build task: %s',
build_task_id, err, exc_info=True) build_task_id, err, exc_info=True)