From 313d4a4d2a691549e3c0762782038020dbd7d359 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 16 Mar 2023 22:48:35 +0100 Subject: [PATCH] db: debug of update feature --- build_analytics/build_analytics/db.py | 42 ++++++++++++------- .../build_analytics/extractor/extractor.py | 5 ++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index 35200f4..edbfe4c 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -10,7 +10,6 @@ from .models.build_node_stat_db import BuildNodeStatDB from .models.db_config import DbConfig from .models.web_node_stat_db import WebNodeStatDB from .models.test_task_db import TestTaskDB -from .models.test_step_stat_db import TestStepStatDB class DB(): @@ -109,9 +108,10 @@ class DB(): res: Dict[int, Dict[int, int]] = {} # getting unfinished builds - sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s ;' - cur = self.__conn.cursor(not_before.timestamp()) - cur.execute(sql) + sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;' + cur = self.__conn.cursor() + cur.execute(sql, (not_before.timestamp(),)) + logging.debug('raw SQL query: %s', cur.query) for row in cur.fetchall(): res[row[0]] = {} @@ -163,7 +163,8 @@ class DB(): 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id) if self.stat_exists(task_id=stat.build_task_id, stat_name_id=stat.stat_name_id, - table_name='web_node_stats'): + table_name='web_node_stats', + column_name='build_task_id'): sql = ''' UPDATE web_node_stats SET start_ts = %(start_ts)s, end_ts = %(end_ts)s @@ -187,7 +188,8 @@ class DB(): 'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id) if self.stat_exists(task_id=stat.build_task_id, stat_name_id=stat.stat_name_id, - table_name='build_node_stats'): + table_name='build_node_stats', + column_name='build_task_id'): sql = ''' UPDATE build_node_stats SET start_ts = %(start_ts)s, end_ts = %(end_ts)s @@ -221,11 +223,11 @@ class DB(): return None return int(val[0]) - def stat_exists(self, task_id: int, stat_name_id: int, table_name: str) -> bool: + def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool: sql = f''' - SELECT COUNT(build_task_id) + SELECT COUNT({column_name}) FROM {table_name} - WHERE build_task_id = %s AND stat_name_id = %s; + WHERE {column_name} = %s AND stat_name_id = %s; ''' cur = self.__conn.cursor() cur.execute(sql, (task_id, stat_name_id)) @@ -263,13 +265,14 @@ class DB(): ''' cur = self.__conn.cursor() sql = ''' - SELECT bt.id + SELECT DISTINCT bt.id FROM build_tasks as bt INNER JOIN test_tasks AS tt ON bt.id = tt.build_task_id - WHERE tt.status_id < 3 AND tt.started_at > %s; + WHERE tt.status_id < 3 AND bt.started_at > %s; ''' - cur.execute(sql, (not_before,)) + cur.execute(sql, (not_before.timestamp(),)) + logging.info('raw SQL query: %s', cur.query) result = [int(row[0]) for row in cur.fetchall()] return result @@ -286,18 +289,27 @@ class DB(): ''' cur.execute(sql, (task.revision, task.status_id, task.started_at, task.id)) + assert cur.rowcount == 1 # test step if not task.steps_stats: continue for s in task.steps_stats: - if self.stat_exists(s.test_task_id, s.stat_name_id, 'test_steps_stats'): + logging.info('test_task_id %s, stat_name_id %s', + s.test_task_id, s.stat_name_id) + if self.stat_exists(s.test_task_id, + s.stat_name_id, + 'test_steps_stats', + 'test_task_id'): sql = ''' UPDATE test_steps_stats SET start_ts = %s, - SET finish_ts = %s; + finish_ts = %s + WHERE test_task_id = %s AND stat_name_id = %s; ''' - cur.execute(sql, s.start_ts, s.finish_ts) + cur.execute(sql, (s.start_ts, s.finish_ts, + s.test_task_id, s.stat_name_id)) + assert cur.rowcount == 1 else: sql = ''' INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts) diff --git a/build_analytics/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py index df83acc..45770bc 100644 --- a/build_analytics/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -102,7 +102,8 @@ class Extractor: def update_builds(self): logging.info('Getting list of tasks from DB') - unfinished_tasks = self.db.get_unfinished_builds() + unfinished_tasks = self.db.get_unfinished_builds( + self.config.oldest_to_update) for build_id, build_tasks_db in unfinished_tasks.items(): try: logging.info('Getting status of build %d', build_id) @@ -136,7 +137,7 @@ class Extractor: logging.info('updating test tasks') tasks_db = [t.as_db_model() for t in tasks_api] self.db.update_test_tasks(tasks_db) - except Exception as err: + except Exception as err: # pylint: disable=broad-except logging.error( 'failed to update tests for %d build task: %s', build_task_id, err, exc_info=True)