kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

db: debug of update feature

Browse Source
This commit is contained in:
Kirill Zhukov 2023-03-16 22:48:35 +01:00
parent 679328093a
commit 313d4a4d2a
2 changed files with 30 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
build_analytics/build_analytics/db.py
View File

@ -10,7 +10,6 @@ from .models.build_node_stat_db import BuildNodeStatDB
from .models.db_config import DbConfig
from .models.web_node_stat_db import WebNodeStatDB
from .models.test_task_db import TestTaskDB
from .models.test_step_stat_db import TestStepStatDB
class DB():
@ -109,9 +108,10 @@ class DB():
res: Dict[int, Dict[int, int]] = {}
# getting unfinished builds
sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s ;'
cur = self.__conn.cursor(not_before.timestamp())
cur.execute(sql)
sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
cur = self.__conn.cursor()
cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
for row in cur.fetchall():
res[row[0]] = {}
@ -163,7 +163,8 @@ class DB():
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='web_node_stats'):
table_name='web_node_stats',
column_name='build_task_id'):
sql = '''
UPDATE web_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -187,7 +188,8 @@ class DB():
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='build_node_stats'):
table_name='build_node_stats',
column_name='build_task_id'):
sql = '''
UPDATE build_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -221,11 +223,11 @@ class DB():
return None
return int(val[0])
def stat_exists(self, task_id: int, stat_name_id: int, table_name: str) -> bool:
def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool:
sql = f'''
SELECT COUNT(build_task_id)
SELECT COUNT({column_name})
FROM {table_name}
WHERE build_task_id = %s AND stat_name_id = %s;
WHERE {column_name} = %s AND stat_name_id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (task_id, stat_name_id))
@ -263,13 +265,14 @@ class DB():
'''
cur = self.__conn.cursor()
sql = '''
SELECT bt.id
SELECT DISTINCT bt.id
FROM build_tasks as bt
INNER JOIN test_tasks AS tt
ON bt.id = tt.build_task_id
WHERE tt.status_id < 3 AND tt.started_at > %s;
WHERE tt.status_id < 3 AND bt.started_at > %s;
'''
cur.execute(sql, (not_before,))
cur.execute(sql, (not_before.timestamp(),))
logging.info('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()]
return result
@ -286,18 +289,27 @@ class DB():
'''
cur.execute(sql, (task.revision, task.status_id,
task.started_at, task.id))
assert cur.rowcount == 1
# test step
if not task.steps_stats:
continue
for s in task.steps_stats:
if self.stat_exists(s.test_task_id, s.stat_name_id, 'test_steps_stats'):
logging.info('test_task_id %s, stat_name_id %s',
s.test_task_id, s.stat_name_id)
if self.stat_exists(s.test_task_id,
s.stat_name_id,
'test_steps_stats',
'test_task_id'):
sql = '''
UPDATE test_steps_stats
SET start_ts = %s,
SET finish_ts = %s;
finish_ts = %s
WHERE test_task_id = %s AND stat_name_id = %s;
'''
cur.execute(sql, s.start_ts, s.finish_ts)
cur.execute(sql, (s.start_ts, s.finish_ts,
s.test_task_id, s.stat_name_id))
assert cur.rowcount == 1
else:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)

5
build_analytics/build_analytics/extractor/extractor.py
View File

@ -102,7 +102,8 @@ class Extractor:
def update_builds(self):
logging.info('Getting list of tasks from DB')
unfinished_tasks = self.db.get_unfinished_builds()
unfinished_tasks = self.db.get_unfinished_builds(
self.config.oldest_to_update)
for build_id, build_tasks_db in unfinished_tasks.items():
try:
logging.info('Getting status of build %d', build_id)
@ -136,7 +137,7 @@ class Extractor:
logging.info('updating test tasks')
tasks_db = [t.as_db_model() for t in tasks_api]
self.db.update_test_tasks(tasks_db)
except Exception as err:
except Exception as err: # pylint: disable=broad-except
logging.error(
'failed to update tests for %d build task: %s',
build_task_id, err, exc_info=True)