build_analytics/build_analytics/api_client.py

@@ -143,14 +143,22 @@ class APIclient():
         return Build(**params)
 
     def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
-        ep = f'/api/v1/tests/{build_task_id}/latest'
+        result: List[TestTask] = []
-        url = urljoin(self.api_root, ep)
+        revision = 1
-        headers = {'accept': 'application/json'}
+        while True:
+            ep = f'/api/v1/tests/{build_task_id}/{revision}'
+            url = urljoin(self.api_root, ep)
+            headers = {'accept': 'application/json'}
 
-        response = requests.get(
+            response = requests.get(
-            url, headers=headers, timeout=self.timeout)
+                url, headers=headers, timeout=self.timeout)
-        response.raise_for_status()
+            response.raise_for_status()
-        return self.__parse_test_tasks(response.json(), build_task_id)
+            raw_tasks = response.json()
+            if len(raw_tasks) == 0:
+                break
+            result = result + self.__parse_test_tasks(raw_tasks, build_task_id)
+            revision += 1
+        return result
 
     def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
                            build_task_id: int,

build_analytics/build_analytics/db.py

@@ -26,14 +26,15 @@ class DB():
     def __del__(self):
         self.close_conn()
 
-    def build_exists(self, build_id: int) -> bool:
+    def row_exists(self, pk: int, table: str) -> bool:
+        assert table in ['builds', 'test_tasks']
         sql = f'''
-                SELECT COUNT(id)
+              SELECT COUNT(id)
-                FROM builds
+              FROM {table}
-                WHERE id = %s;
+              WHERE id = %s;
-                '''
+              '''
         cur = self.__conn.cursor()
-        cur.execute(sql, (build_id,))
+        cur.execute(sql, (pk,))
         val = int(cur.fetchone()[0])
         return val == 1
 
@@ -245,31 +246,6 @@ class DB():
         val = int(cur.fetchone()[0])
         return val == 1
 
-    def insert_test_task(self, task: TestTaskDB):
-        cur = self.__conn.cursor()
-        # inserting test task itself
-        sql = '''
-                INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
-                VALUES
-                  (%s, %s, %s, %s, %s, %s);
-              '''
-        cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
-                          task.package_fullname, task.started_at))
-
-        if task.steps_stats:
-            # inserting  test steps stats
-            for ss in task.steps_stats:
-                sql = '''
-                    INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
-                    VALUES
-                        (%s, %s, %s, %s);
-                    '''
-                cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
-                            ss.start_ts, ss.finish_ts))
-
-        # commiting changes
-        self.__conn.commit()
-
     def get_build_tasks_for_unfinished_tests(self, not_before: datetime) -> List[int]:
         '''
         getting build tasks id of unfinished test tasks
@@ -287,27 +263,37 @@ class DB():
         result = [int(row[0]) for row in cur.fetchall()]
         return result
 
-    def update_test_tasks(self, test_tasks: List[TestTaskDB]):
+    def insert_update_test_tasks(self, test_tasks: List[TestTaskDB]):
         cur = self.__conn.cursor()
         # test tasks
         for task in test_tasks:
-            sql = '''
+            if self.row_exists(pk=task.id, table='test_tasks'):
-                UPDATE test_tasks
+                sql = '''
-                SET revision = %s,
+                    UPDATE test_tasks
-                    status_id = %s,
+                    SET revision = %s,
-                    started_at = %s
+                        status_id = %s,
-                WHERE id = %s;
+                        started_at = %s
-              '''
+                    WHERE id = %s;
-            cur.execute(sql, (task.revision, task.status_id,
+                '''
-                        task.started_at, task.id))
+                cur.execute(sql, (task.revision, task.status_id,
-            assert cur.rowcount == 1
+                            task.started_at, task.id))
+                assert cur.rowcount == 1
+            else:
+                sql = '''
+                      INSERT INTO test_tasks(
+                        id, build_task_id, revision, status_id, package_fullname, started_at)
+                      VALUES
+                      (%s, %s, %s, %s, %s, %s);
+                      '''
+                cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
+                                  task.package_fullname, task.started_at))
 
             # test step
             if not task.steps_stats:
                 continue
             for s in task.steps_stats:
-                logging.info('test_task_id %s, stat_name_id %s',
+                logging.debug('test_task_id %s, stat_name_id %s',
-                             s.test_task_id, s.stat_name_id)
+                              s.test_task_id, s.stat_name_id)
                 if self.stat_exists(s.test_task_id,
                                     s.stat_name_id,
                                     'test_steps_stats',
@@ -323,7 +309,8 @@ class DB():
                     assert cur.rowcount == 1
                 else:
                     sql = '''
-                          INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
+                          INSERT INTO test_steps_stats (
+                            test_task_id, stat_name_id, start_ts, finish_ts)
                           VALUES (%s, %s, %s, %s);
                           '''
                     cur.execute(sql, (s.test_task_id, s.stat_name_id,

build_analytics/build_analytics/extractor/extractor.py

@@ -35,7 +35,7 @@ class Extractor:
                     break
 
                 # some builds could move from one page to another
-                if self.db.build_exists(build_id=build.id):
+                if self.db.row_exists(pk=build.id, table='builds'):
                     continue
 
                 # inserting build build tasks and build tasks statistics
@@ -64,10 +64,10 @@ class Extractor:
                         'getting test tasks for build task %s', build_task.id)
                     test_tasks = self.api.get_test_tasks(build_task.id)
                     logging.info('received %d tests tasks', len(test_tasks))
-                    for t in test_tasks:
+                    if len(test_tasks) > 0:
-                        logging.info(
+                        logging.info('inserting test tasks')
-                            'build task %s: inserting test task %s', build_task.id, t.id)
+                        as_db = [t.as_db_model() for t in test_tasks]
-                        self.db.insert_test_task(t.as_db_model())
+                        self.db.insert_update_test_tasks(as_db)
                 build_count += 1
             page_num += 1
         return build_count
@@ -140,7 +140,7 @@ class Extractor:
                 tasks_api = self.api.get_test_tasks(build_task_id)
                 logging.info('updating test tasks')
                 tasks_db = [t.as_db_model() for t in tasks_api]
-                self.db.update_test_tasks(tasks_db)
+                self.db.insert_update_test_tasks(tasks_db)
             except Exception as err:  # pylint: disable=broad-except
                 logging.error(
                     'failed to update tests for %d build task: %s',

build_analytics/config_default.yml

@@ -62,10 +62,14 @@ scrape_interval: 3600
 # default: 5808 (first build with correct metrics)
 start_from:
 
-
 # oldest_to_update
 # oldest (in days) unfinished object (build/task/step...) that we will try to update
 # required: false
 # default: 7
 oldest_to_update_days: 7
 
+# api_timeout
+# how long (in seconds) we will wait for API response
+# required: false
+# default: 30
+api_timeout: 30

build_analytics/migrations/3.sql

@@ -68,7 +68,10 @@ CREATE INDEX test_steps_stats_start_ts
 ON test_steps_stats(start_ts);
 
 CREATE INDEX test_steps_stats_end_ts
-ON test_steps_stats(end_ts);
+ON test_steps_stats(finish_ts);
+
+-- increasing size of name field
+ALTER TABLE build_tasks ALTER COLUMN name TYPE varchar(150);
 
 
 UPDATE schema_version