From 3ed0dcdd786bd2db042917daeb5255b3250807cc Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 10 Mar 2023 19:35:12 +0100 Subject: [PATCH] added build step: multilib_processing db: stop adding empty build steps (without start_ts) --- build_analytics/build_analytics/api_client.py | 3 +- build_analytics/build_analytics/const.py | 1 + build_analytics/build_analytics/db.py | 83 +++++++++++++++---- .../build_analytics/models/web_node_stats.py | 1 + build_analytics/migrations/2.sql | 13 +++ 5 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 build_analytics/migrations/2.sql diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index f30adbd..56a293b 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -72,7 +72,8 @@ class APIclient(): return build_node_stats def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats: - keys = ['build_done', 'logs_processing', 'packages_processing'] + keys = ['build_done', 'logs_processing', + 'packages_processing', 'multilib_processing'] params = {} logging.debug('raw json: %s', stats) for k in keys: diff --git a/build_analytics/build_analytics/const.py b/build_analytics/build_analytics/const.py index c29d1d9..c9c9370 100644 --- a/build_analytics/build_analytics/const.py +++ b/build_analytics/build_analytics/const.py @@ -27,6 +27,7 @@ class WebNodeStatsEnum(IntEnum): build_done = 0 logs_processing = 1 packages_processing = 2 + multilib_processing = 3 class BuildNodeStatsEnum(IntEnum): diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index 68d6c62..b266249 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -50,6 +50,11 @@ class DB(): # inserting web node stats for stat in web_node_stats: + + # do not insert empty stats + if stat.start_ts is None: + continue + sql = ''' INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts) VALUES (%s, %s, %s, %s); @@ -57,10 +62,15 @@ class DB(): cur.execute(sql, (stat.build_task_id, stat.stat_name_id, stat.start_ts, stat.end_ts)) logging.debug('raw SQL query: %s', cur.query) + self.__conn.commit() # inserting build node stats for stat in build_node_stats: - logging.debug('BuildNodeStats: %s', stat) + + # do not insert empty stats + if stat.start_ts is None: + continue + sql = ''' INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) VALUES (%s, %s, %s, %s); @@ -134,7 +144,6 @@ class DB(): build_node_stats: List[BuildNodeStatDB]): cur = self.__conn.cursor() - # updating build_task sql = ''' UPDATE build_tasks SET status_id = %s, @@ -144,28 +153,55 @@ class DB(): ''' cur.execute(sql, (build_task.status_id, build_task.started_at, build_task.finished_at, build_task.id)) + logging.debug('raw SQL query: %s', cur.query) # updating web_node_stats for stat in web_node_stats: - sql = ''' - UPDATE web_node_stats - SET start_ts = %s, - end_ts = %s - WHERE build_task_id = %s AND stat_name_id = %s; - ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, - build_task.id, stat.stat_name_id)) + logging.debug( + 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id) + if self.stat_exists(build_task_id=stat.build_task_id, + stat_name_id=stat.stat_name_id, + table_name='web_node_stats'): + sql = ''' + UPDATE web_node_stats + SET start_ts = %(start_ts)s, end_ts = %(end_ts)s + WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s + ''' + else: + sql = ''' + INSERT INTO web_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s); + ''' + params = {'build_task_id': build_task.id, + 'stat_name_id': stat.stat_name_id, + 'start_ts': stat.start_ts, + 'end_ts': stat.end_ts} + cur.execute(sql, params) + logging.debug('raw SQL query: %s', cur.query) # updating build_node_stats for stat in build_node_stats: - sql = ''' - UPDATE build_node_stats - SET start_ts = %s, - end_ts = %s - WHERE build_task_id = %s and stat_name_id = %s; - ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, - build_task.id, stat.stat_name_id)) + logging.debug( + 'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id) + if self.stat_exists(build_task_id=stat.build_task_id, + stat_name_id=stat.stat_name_id, + table_name='build_node_stats'): + sql = ''' + UPDATE build_node_stats + SET start_ts = %(start_ts)s, end_ts = %(end_ts)s + WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s + ''' + else: + sql = ''' + INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s); + ''' + params = {'build_task_id': build_task.id, + 'stat_name_id': stat.stat_name_id, + 'start_ts': stat.start_ts, + 'end_ts': stat.end_ts} + logging.debug('raw SQL query: %s', cur.query) + cur.execute(sql, params) # commiting changes self.__conn.commit() @@ -182,3 +218,14 @@ class DB(): if not val: return None return int(val[0]) + + def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool: + sql = f''' + SELECT COUNT(build_task_id) + FROM {table_name} + WHERE build_task_id = %s AND stat_name_id = %s; + ''' + cur = self.__conn.cursor() + cur.execute(sql, (build_task_id, stat_name_id)) + val = int(cur.fetchone()[0]) + return val == 1 diff --git a/build_analytics/build_analytics/models/web_node_stats.py b/build_analytics/build_analytics/models/web_node_stats.py index 0476863..504ccea 100644 --- a/build_analytics/build_analytics/models/web_node_stats.py +++ b/build_analytics/build_analytics/models/web_node_stats.py @@ -15,6 +15,7 @@ class WebNodeStats(BaseModel): build_done: BuildStat logs_processing: BuildStat packages_processing: BuildStat + multilib_processing: BuildStat def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]: result = [] diff --git a/build_analytics/migrations/2.sql b/build_analytics/migrations/2.sql new file mode 100644 index 0000000..1b10c5f --- /dev/null +++ b/build_analytics/migrations/2.sql @@ -0,0 +1,13 @@ +BEGIN; + +INSERT INTO web_node_stats_enum (id, value) +VALUES + (3, 'multilib_processing'); + +ALTER TABLE web_node_stats +ADD CONSTRAINT web_node_stats_unique UNIQUE (build_task_id, stat_name_id); + +ALTER TABLE build_node_stats +ADD CONSTRAINT build_node_stats_unique UNIQUE (build_task_id, stat_name_id); + +COMMIT; \ No newline at end of file