build_analytics/build_analytics/api_client.py

@@ -143,22 +143,14 @@ class APIclient():
         return Build(**params)
 
     def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
-        result: List[TestTask] = []
-        revision = 1
-        while True:
-            ep = f'/api/v1/tests/{build_task_id}/{revision}'
-            url = urljoin(self.api_root, ep)
-            headers = {'accept': 'application/json'}
+        ep = f'/api/v1/tests/{build_task_id}/latest'
+        url = urljoin(self.api_root, ep)
+        headers = {'accept': 'application/json'}
 
-            response = requests.get(
-                url, headers=headers, timeout=self.timeout)
-            response.raise_for_status()
-            raw_tasks = response.json()
-            if len(raw_tasks) == 0:
-                break
-            result = result + self.__parse_test_tasks(raw_tasks, build_task_id)
-            revision += 1
-        return result
+        response = requests.get(
+            url, headers=headers, timeout=self.timeout)
+        response.raise_for_status()
+        return self.__parse_test_tasks(response.json(), build_task_id)
 
     def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
                            build_task_id: int,

build_analytics/build_analytics/db.py

@@ -26,15 +26,14 @@ class DB():
     def __del__(self):
         self.close_conn()
 
-    def row_exists(self, pk: int, table: str) -> bool:
-        assert table in ['builds', 'test_tasks']
+    def build_exists(self, build_id: int) -> bool:
         sql = f'''
-              SELECT COUNT(id)
-              FROM {table}
-              WHERE id = %s;
-              '''
+                SELECT COUNT(id)
+                FROM builds
+                WHERE id = %s;
+                '''
         cur = self.__conn.cursor()
-        cur.execute(sql, (pk,))
+        cur.execute(sql, (build_id,))
         val = int(cur.fetchone()[0])
         return val == 1
 
@@ -246,6 +245,31 @@ class DB():
         val = int(cur.fetchone()[0])
         return val == 1
 
+    def insert_test_task(self, task: TestTaskDB):
+        cur = self.__conn.cursor()
+        # inserting test task itself
+        sql = '''
+                INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
+                VALUES
+                  (%s, %s, %s, %s, %s, %s);
+              '''
+        cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
+                          task.package_fullname, task.started_at))
+
+        if task.steps_stats:
+            # inserting  test steps stats
+            for ss in task.steps_stats:
+                sql = '''
+                    INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
+                    VALUES
+                        (%s, %s, %s, %s);
+                    '''
+                cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
+                            ss.start_ts, ss.finish_ts))
+
+        # commiting changes
+        self.__conn.commit()
+
     def get_build_tasks_for_unfinished_tests(self, not_before: datetime) -> List[int]:
         '''
         getting build tasks id of unfinished test tasks
@@ -263,37 +287,27 @@ class DB():
         result = [int(row[0]) for row in cur.fetchall()]
         return result
 
-    def insert_update_test_tasks(self, test_tasks: List[TestTaskDB]):
+    def update_test_tasks(self, test_tasks: List[TestTaskDB]):
         cur = self.__conn.cursor()
         # test tasks
         for task in test_tasks:
-            if self.row_exists(pk=task.id, table='test_tasks'):
-                sql = '''
-                    UPDATE test_tasks
-                    SET revision = %s,
-                        status_id = %s,
-                        started_at = %s
-                    WHERE id = %s;
-                '''
-                cur.execute(sql, (task.revision, task.status_id,
-                            task.started_at, task.id))
-                assert cur.rowcount == 1
-            else:
-                sql = '''
-                      INSERT INTO test_tasks(
-                        id, build_task_id, revision, status_id, package_fullname, started_at)
-                      VALUES
-                      (%s, %s, %s, %s, %s, %s);
-                      '''
-                cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
-                                  task.package_fullname, task.started_at))
+            sql = '''
+                UPDATE test_tasks
+                SET revision = %s,
+                    status_id = %s,
+                    started_at = %s
+                WHERE id = %s;
+              '''
+            cur.execute(sql, (task.revision, task.status_id,
+                        task.started_at, task.id))
+            assert cur.rowcount == 1
 
             # test step
             if not task.steps_stats:
                 continue
             for s in task.steps_stats:
-                logging.debug('test_task_id %s, stat_name_id %s',
-                              s.test_task_id, s.stat_name_id)
+                logging.info('test_task_id %s, stat_name_id %s',
+                             s.test_task_id, s.stat_name_id)
                 if self.stat_exists(s.test_task_id,
                                     s.stat_name_id,
                                     'test_steps_stats',
@@ -309,8 +323,7 @@ class DB():
                     assert cur.rowcount == 1
                 else:
                     sql = '''
-                          INSERT INTO test_steps_stats (
-                            test_task_id, stat_name_id, start_ts, finish_ts)
+                          INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
                           VALUES (%s, %s, %s, %s);
                           '''
                     cur.execute(sql, (s.test_task_id, s.stat_name_id,

build_analytics/build_analytics/extractor/extractor.py

@@ -35,7 +35,7 @@ class Extractor:
                     break
 
                 # some builds could move from one page to another
-                if self.db.row_exists(pk=build.id, table='builds'):
+                if self.db.build_exists(build_id=build.id):
                     continue
 
                 # inserting build build tasks and build tasks statistics
@@ -64,10 +64,10 @@ class Extractor:
                         'getting test tasks for build task %s', build_task.id)
                     test_tasks = self.api.get_test_tasks(build_task.id)
                     logging.info('received %d tests tasks', len(test_tasks))
-                    if len(test_tasks) > 0:
-                        logging.info('inserting test tasks')
-                        as_db = [t.as_db_model() for t in test_tasks]
-                        self.db.insert_update_test_tasks(as_db)
+                    for t in test_tasks:
+                        logging.info(
+                            'build task %s: inserting test task %s', build_task.id, t.id)
+                        self.db.insert_test_task(t.as_db_model())
                 build_count += 1
             page_num += 1
         return build_count
@@ -140,7 +140,7 @@ class Extractor:
                 tasks_api = self.api.get_test_tasks(build_task_id)
                 logging.info('updating test tasks')
                 tasks_db = [t.as_db_model() for t in tasks_api]
-                self.db.insert_update_test_tasks(tasks_db)
+                self.db.update_test_tasks(tasks_db)
             except Exception as err:  # pylint: disable=broad-except
                 logging.error(
                     'failed to update tests for %d build task: %s',

build_analytics/config_default.yml

@@ -62,14 +62,10 @@ scrape_interval: 3600
 # default: 5808 (first build with correct metrics)
 start_from:
 
+
 # oldest_to_update
 # oldest (in days) unfinished object (build/task/step...) that we will try to update
 # required: false
 # default: 7
 oldest_to_update_days: 7
 
-# api_timeout
-# how long (in seconds) we will wait for API response
-# required: false
-# default: 30
-api_timeout: 30

build_analytics/migrations/3.sql

@@ -68,10 +68,7 @@ CREATE INDEX test_steps_stats_start_ts
 ON test_steps_stats(start_ts);
 
 CREATE INDEX test_steps_stats_end_ts
-ON test_steps_stats(finish_ts);
-
--- increasing size of name field
-ALTER TABLE build_tasks ALTER COLUMN name TYPE varchar(150);
+ON test_steps_stats(end_ts);
 
 
 UPDATE schema_version