kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

ALBS-1026: add statistics for each build_task step #1

Merged
kzhukov merged 12 commits from ALBS-1026 into main 2023-03-15 10:25:53 +00:00
Conversation 0 Commits 12 Files Changed 36 +82 -19
5 changed files with 82 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3ed0dcdd78 - Show all commits

3
build_analytics/build_analytics/api_client.py
View File

@ -72,7 +72,8 @@ class APIclient():
return build_node_stats
def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats:
keys = ['build_done', 'logs_processing', 'packages_processing']
keys = ['build_done', 'logs_processing',
'packages_processing', 'multilib_processing']
params = {}
logging.debug('raw json: %s', stats)
for k in keys:

1
build_analytics/build_analytics/const.py
View File

@ -27,6 +27,7 @@ class WebNodeStatsEnum(IntEnum):
build_done = 0
logs_processing = 1
packages_processing = 2
multilib_processing = 3
class BuildNodeStatsEnum(IntEnum):

83
build_analytics/build_analytics/db.py
View File

@ -50,6 +50,11 @@ class DB():
# inserting web node stats
for stat in web_node_stats:
# do not insert empty stats
if stat.start_ts is None:
continue
sql = '''
INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts)
VALUES (%s, %s, %s, %s);
@ -57,10 +62,15 @@ class DB():
cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
stat.start_ts, stat.end_ts))
logging.debug('raw SQL query: %s', cur.query)
self.__conn.commit()
# inserting build node stats
for stat in build_node_stats:
logging.debug('BuildNodeStats: %s', stat)
# do not insert empty stats
if stat.start_ts is None:
continue
sql = '''
INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
VALUES (%s, %s, %s, %s);
@ -134,7 +144,6 @@ class DB():
build_node_stats: List[BuildNodeStatDB]):
cur = self.__conn.cursor()
# updating build_task
sql = '''
UPDATE build_tasks
SET status_id = %s,
@ -144,28 +153,55 @@ class DB():
'''
cur.execute(sql, (build_task.status_id, build_task.started_at,
build_task.finished_at, build_task.id))
logging.debug('raw SQL query: %s', cur.query)
# updating web_node_stats
for stat in web_node_stats:
sql = '''
UPDATE web_node_stats
SET start_ts = %s,
end_ts = %s
WHERE build_task_id = %s AND stat_name_id = %s;
'''
cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
logging.debug(
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='web_node_stats'):
sql = '''
UPDATE web_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
'''
else:
sql = '''
INSERT INTO web_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
'''
params = {'build_task_id': build_task.id,
'stat_name_id': stat.stat_name_id,
'start_ts': stat.start_ts,
'end_ts': stat.end_ts}
cur.execute(sql, params)
logging.debug('raw SQL query: %s', cur.query)
# updating build_node_stats
for stat in build_node_stats:
sql = '''
UPDATE build_node_stats
SET start_ts = %s,
end_ts = %s
WHERE build_task_id = %s and stat_name_id = %s;
'''
cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
logging.debug(
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='build_node_stats'):
sql = '''
UPDATE build_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
'''
else:
sql = '''
INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
'''
params = {'build_task_id': build_task.id,
'stat_name_id': stat.stat_name_id,
'start_ts': stat.start_ts,
'end_ts': stat.end_ts}
logging.debug('raw SQL query: %s', cur.query)
cur.execute(sql, params)
# commiting changes
self.__conn.commit()
@ -182,3 +218,14 @@ class DB():
if not val:
return None
return int(val[0])
def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool:
sql = f'''
SELECT COUNT(build_task_id)
FROM {table_name}
WHERE build_task_id = %s AND stat_name_id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (build_task_id, stat_name_id))
val = int(cur.fetchone()[0])
return val == 1

1
build_analytics/build_analytics/models/web_node_stats.py
View File

@ -15,6 +15,7 @@ class WebNodeStats(BaseModel):
build_done: BuildStat
logs_processing: BuildStat
packages_processing: BuildStat
multilib_processing: BuildStat
def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]:
result = []

13
build_analytics/migrations/2.sql Normal file
View File

@ -0,0 +1,13 @@
BEGIN;
INSERT INTO web_node_stats_enum (id, value)
VALUES
(3, 'multilib_processing');
ALTER TABLE web_node_stats
ADD CONSTRAINT web_node_stats_unique UNIQUE (build_task_id, stat_name_id);
ALTER TABLE build_node_stats
ADD CONSTRAINT build_node_stats_unique UNIQUE (build_task_id, stat_name_id);
COMMIT;