build_analytics/build_analytics/api_client.py

@@ -150,7 +150,7 @@ class APIclient():
         response = requests.get(
             url, headers=headers, timeout=self.timeout)
         response.raise_for_status()
-        return self.__parse_test_tasks()
+        return self.__parse_test_tasks(response.json(), build_task_id)
 
     def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
                            build_task_id: int,
@@ -161,33 +161,30 @@ class APIclient():
                 started_raw = task['alts_response']['stats']['started_at']
                 started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
                 stats_raw = task['alts_response']['stats']
-                results_raw = task['results']['tests']
-                step_stats = self.__parse_test_steps_stats(
-                    stats_raw, results_raw)
+                steps_stats = self.__parse_test_steps_stats(stats_raw)
             else:
                 started_at = None
-                step_stats = None
+                steps_stats = None
             params = {
                 'id': task['id'],
                 'build_task_id': build_task_id,
                 'revision': task['revision'],
                 'status': task['status'],
-                'package_fullname': '_'.join([raw_tasks['package_name'],
-                                              raw_tasks['package_version'],
-                                              raw_tasks['package_release']]),
+                'package_fullname': '_'.join([task['package_name'],
+                                              task['package_version'],
+                                              task['package_release']]),
                 'started_at': started_at,
-                'step_stats': step_stats
+                'steps_stats': steps_stats
             }
             result.append(TestTask(**params))
         return result
 
-    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats:
+    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
         teast_steps_params = {}
         for field_name in TestStepsStats.__fields__.keys():
             try:
                 p = stats_raw[field_name]
             except KeyError:
                 continue
-            p['success'] = results_raw[field_name]['success']
             teast_steps_params[field_name] = TestStepStat(**p)
         return TestStepsStats(**teast_steps_params)

build_analytics/build_analytics/const.py

@@ -54,7 +54,8 @@ class TestStepEnum(IntEnum):
     install_package = 0
     stop_enviroment = 1
     initial_provision = 2
-    start_enviroment = 3
+    start_environment = 3
     uninstall_package = 4
     initialize_terraform = 5
     package_integrity_tests = 6
+    stop_environment = 7

build_analytics/build_analytics/db.py

@@ -232,24 +232,26 @@ class DB():
         return val == 1
 
     def insert_test_task(self, task: TestTaskDB):
+        cur = self.__conn.cursor()
         # inserting test task itself
         sql = '''
                 INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
-                VALUES (%s, %s, %s, %s, %s, %s);
+                VALUES
+                  (%s, %s, %s, %s, %s, %s);
               '''
-
-        cur = self.__conn.cursor()
-        cur.execute(sql, (task.id, task.build_task_id, task.status_id,
+        cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
                           task.package_fullname, task.started_at))
+
+        if task.steps_stats:
             # inserting  test steps stats
             for ss in task.steps_stats:
                 sql = '''
-                  INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
+                    INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
                     VALUES
-                    (%s, %s, %s, %s, %s);
+                        (%s, %s, %s, %s);
                     '''
                 cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
-                        ss.start_ts, ss.end_ts, ss.success))
+                            ss.start_ts, ss.finish_ts))
 
         # commiting changes
         self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -56,6 +56,15 @@ class Extractor:
                     except Exception as error:  # pylint: disable=broad-except
                         logging.error('build %s: failed to insert build task %d: %s',
                                       build.id, build_task.id, error, exc_info=True)
+
+                    logging.info(
+                        'getting test tasks for build task %s', build_task.id)
+                    test_tasks = self.api.get_test_tasks(build_task.id)
+                    logging.info('received %d tests tasks', len(test_tasks))
+                    for t in test_tasks:
+                        logging.info(
+                            'build task %s: inserting test task %s', build_task.id, t.id)
+                        self.db.insert_test_task(t.as_db_model())
                 build_count += 1
             page_num += 1
         return build_count

build_analytics/build_analytics/models/sign_task_db.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
 
 
 class SignTaskDB(BaseModel):

build_analytics/build_analytics/models/test_step_stat.py

@@ -6,5 +6,4 @@ from pydantic import BaseModel  # pylint: disable=no-name-in-module
 
 class TestStepStat(BaseModel):
     start_ts: Optional[datetime] = None
-    end_ts: Optional[datetime] = None
-    success: bool
+    finish_ts: Optional[datetime] = None

build_analytics/build_analytics/models/test_step_stat_db.py

@@ -6,5 +6,4 @@ class TestStepStatDB(BaseModel):
     test_task_id: int
     stat_name_id: int
     start_ts: Optional[float] = None
-    end_ts: Optional[float] = None
-    success: bool
+    finish_ts: Optional[float] = None

build_analytics/build_analytics/models/test_steps_stats.py

@@ -24,14 +24,13 @@ class TestStepsStats(BaseModel):
                 continue
             start_ts = stats.start_ts.timestamp() \
                 if stats.start_ts else None
-            end_ts = stats.end_ts.timestamp() \
-                if stats.end_ts else None
+            finish_ts = stats.finish_ts.timestamp() \
+                if stats.finish_ts else None
             stat_name_id = TestStepEnum[field_name].value
 
             test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
                                                stat_name_id=stat_name_id,
                                                start_ts=start_ts,
-                                               end_ts=end_ts,
-                                               success=stats.success)
+                                               finish_ts=finish_ts)
             result.append(test_step_stat_db)
         return result

build_analytics/build_analytics/models/test_task.py

@@ -14,7 +14,7 @@ class TestTask(BaseModel):
     status: int
     package_fullname: str
     started_at: Optional[datetime] = None
-    steps_stats: TestStepsStats
+    steps_stats: Optional[TestStepsStats] = None
 
     def as_db_model(self) -> TestTaskDB:
         started_at = self.started_at.timestamp() \
@@ -23,9 +23,9 @@ class TestTask(BaseModel):
             'id': self.id,
             'build_task_id': self.build_task_id,
             'revision': self.revision,
-            'status': self.status,
+            'status_id': self.status,
             'package_fullname': self.package_fullname,
             'started_at': started_at,
-            'steps_stats': self.step_stats.as_db(self.id)
+            'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
         }
         return TestTaskDB(**params)

build_analytics/build_analytics/models/test_task_db.py

@@ -13,5 +13,5 @@ class TestTaskDB(BaseModel):
     revision: int
     status_id: int
     package_fullname: str
-    started_at: float
-    steps_stats: List[TestStepStatDB] = None
+    started_at: Optional[float] = None
+    steps_stats: Optional[List[TestStepStatDB]] = None

build_analytics/migrations/3.sql

@@ -48,17 +48,17 @@ VALUES
     (3, 'start_environment'),
     (4, 'uninstall_package'),
     (5, 'initialize_terraform'),
-    (6, 'package_integrity_tests');
+    (6, 'package_integrity_tests'),
+    (7, 'stop_environment');
 
 
 
 -- test_steps
 CREATE TABLE test_steps_stats(
-    test_task_id INTEGER,
-    stat_name_id  INTEGER REFERENCES (id) ON DELETE SET NULL,
+    test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
+    stat_name_id  INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
     start_ts DOUBLE PRECISION,
-    end_ts DOUBLE PRECISION,
-    success BOOLEAN
+    finish_ts DOUBLE PRECISION
 );
 
 ALTER TABLE test_steps_stats

build_analytics/releases.txt

@@ -8,3 +8,6 @@ First version
 
 0.2.1 (2023-03-15)
  - Added canceled Build task status
+
+0.3.0 (IN PROGRESS)
+ - Added test tasks stats