build_analytics/build_analytics/api_client.py

@@ -72,7 +72,8 @@ class APIclient():
         return build_node_stats
 
     def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats:
-        keys = ['build_done', 'logs_processing', 'packages_processing']
+        keys = ['build_done', 'logs_processing',
+                'packages_processing', 'multilib_processing']
         params = {}
         logging.debug('raw json: %s', stats)
         for k in keys:

build_analytics/build_analytics/const.py

@@ -27,6 +27,7 @@ class WebNodeStatsEnum(IntEnum):
     build_done = 0
     logs_processing = 1
     packages_processing = 2
+    multilib_processing = 3
 
 
 class BuildNodeStatsEnum(IntEnum):

build_analytics/build_analytics/db.py

@@ -50,6 +50,11 @@ class DB():
 
         # inserting web node stats
         for stat in web_node_stats:
+
+            # do not insert empty stats
+            if stat.start_ts is None:
+                continue
+
             sql = '''
                     INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
@@ -57,10 +62,15 @@ class DB():
             cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
                         stat.start_ts, stat.end_ts))
             logging.debug('raw SQL query: %s', cur.query)
+            self.__conn.commit()
 
         # inserting build node stats
         for stat in build_node_stats:
-            logging.debug('BuildNodeStats: %s', stat)
+
+            # do not insert empty stats
+            if stat.start_ts is None:
+                continue
+
             sql = '''
                     INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
@@ -134,7 +144,6 @@ class DB():
                           build_node_stats: List[BuildNodeStatDB]):
         cur = self.__conn.cursor()
 
-        # updating build_task
         sql = '''
                 UPDATE build_tasks
                 SET status_id = %s,
@@ -144,28 +153,55 @@ class DB():
               '''
         cur.execute(sql, (build_task.status_id, build_task.started_at,
                     build_task.finished_at, build_task.id))
+        logging.debug('raw SQL query: %s', cur.query)
 
         # updating web_node_stats
         for stat in web_node_stats:
-            sql = '''
+            logging.debug(
-                    UPDATE web_node_stats
+                'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
-                    SET start_ts = %s,
+            if self.stat_exists(build_task_id=stat.build_task_id,
-                        end_ts = %s
+                                stat_name_id=stat.stat_name_id,
-                    WHERE build_task_id = %s AND stat_name_id = %s;
+                                table_name='web_node_stats'):
-                '''
+                sql = '''
-            cur.execute(sql, (stat.start_ts, stat.end_ts,
+                      UPDATE web_node_stats
-                        build_task.id, stat.stat_name_id))
+                      SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
+                      WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
+                      '''
+            else:
+                sql = '''
+                       INSERT INTO web_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
+                        VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
+                      '''
+            params = {'build_task_id': build_task.id,
+                      'stat_name_id': stat.stat_name_id,
+                      'start_ts': stat.start_ts,
+                      'end_ts': stat.end_ts}
+            cur.execute(sql, params)
+            logging.debug('raw SQL query: %s', cur.query)
 
         # updating build_node_stats
         for stat in build_node_stats:
-            sql = '''
+            logging.debug(
-                UPDATE build_node_stats
+                'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
-                SET start_ts = %s,
+            if self.stat_exists(build_task_id=stat.build_task_id,
-                    end_ts = %s
+                                stat_name_id=stat.stat_name_id,
-                WHERE build_task_id = %s and stat_name_id = %s;
+                                table_name='build_node_stats'):
-                '''
+                sql = '''
-            cur.execute(sql, (stat.start_ts, stat.end_ts,
+                      UPDATE build_node_stats
-                        build_task.id, stat.stat_name_id))
+                      SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
+                      WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
+                      '''
+            else:
+                sql = '''
+                       INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
+                        VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
+                      '''
+            params = {'build_task_id': build_task.id,
+                      'stat_name_id': stat.stat_name_id,
+                      'start_ts': stat.start_ts,
+                      'end_ts': stat.end_ts}
+            logging.debug('raw SQL query: %s', cur.query)
+            cur.execute(sql, params)
 
         # commiting changes
         self.__conn.commit()
@@ -182,3 +218,14 @@ class DB():
         if not val:
             return None
         return int(val[0])
+
+    def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool:
+        sql = f'''
+                SELECT COUNT(build_task_id)
+                FROM {table_name}
+                WHERE build_task_id = %s AND stat_name_id = %s;
+             '''
+        cur = self.__conn.cursor()
+        cur.execute(sql, (build_task_id, stat_name_id))
+        val = int(cur.fetchone()[0])
+        return val == 1

build_analytics/build_analytics/models/web_node_stats.py

@@ -15,6 +15,7 @@ class WebNodeStats(BaseModel):
     build_done: BuildStat
     logs_processing: BuildStat
     packages_processing: BuildStat
+    multilib_processing: BuildStat
 
     def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]:
         result = []

build_analytics/migrations/2.sql

@@ -0,0 +1,13 @@
+BEGIN;
+
+INSERT INTO web_node_stats_enum (id, value)
+VALUES
+    (3, 'multilib_processing');
+
+ALTER TABLE web_node_stats
+ADD CONSTRAINT web_node_stats_unique UNIQUE (build_task_id, stat_name_id);
+
+ALTER TABLE build_node_stats
+ADD CONSTRAINT build_node_stats_unique UNIQUE (build_task_id, stat_name_id);
+
+COMMIT;