debugging

build_analytics/build_analytics/api_client.py

@@ -1,16 +1,19 @@
 from datetime import datetime
 import logging
 from urllib.parse import urljoin
-from typing import Dict, List
+from typing import Dict, List, Any
 
 import requests
 
+
 from .models.build import Build
 from .models.build_task import BuildTask
 from .models.build_node_stats import BuildNodeStats
 from .models.build_stat import BuildStat
 from .models.web_node_stats import WebNodeStats
-
+from .models.test_task import TestTask
+from .models.test_steps_stats import TestStepsStats
+from .models.test_step_stat import TestStepStat
 
 TZ_OFFSET = '+00:00'
 
@@ -138,3 +141,50 @@ class APIclient():
             'build_tasks': build_tasks}
 
         return Build(**params)
+
+    def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
+        ep = f'/api/v1/tests/{build_task_id}/latest'
+        url = urljoin(self.api_root, ep)
+        headers = {'accept': 'application/json'}
+
+        response = requests.get(
+            url, headers=headers, timeout=self.timeout)
+        response.raise_for_status()
+        return self.__parse_test_tasks(response.json(), build_task_id)
+
+    def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
+                           build_task_id: int,
+                           started_at: str = None) -> List[TestTask]:
+        result: List[TestTask] = []
+        for task in raw_tasks:
+            if task['alts_response']:
+                started_raw = task['alts_response']['stats']['started_at']
+                started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
+                stats_raw = task['alts_response']['stats']
+                steps_stats = self.__parse_test_steps_stats(stats_raw)
+            else:
+                started_at = None
+                steps_stats = None
+            params = {
+                'id': task['id'],
+                'build_task_id': build_task_id,
+                'revision': task['revision'],
+                'status': task['status'],
+                'package_fullname': '_'.join([task['package_name'],
+                                              task['package_version'],
+                                              task['package_release']]),
+                'started_at': started_at,
+                'steps_stats': steps_stats
+            }
+            result.append(TestTask(**params))
+        return result
+
+    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
+        teast_steps_params = {}
+        for field_name in TestStepsStats.__fields__.keys():
+            try:
+                p = stats_raw[field_name]
+            except KeyError:
+                continue
+            teast_steps_params[field_name] = TestStepStat(**p)
+        return TestStepsStats(**teast_steps_params)

build_analytics/build_analytics/const.py

@@ -3,7 +3,7 @@
 from enum import IntEnum
 
 # supported schema version
-DB_SCHEMA_VER = 2
+DB_SCHEMA_VER = 3
 
 
 # ENUMS
@@ -41,3 +41,21 @@ class BuildNodeStatsEnum(IntEnum):
     build_node_task = 6
     cas_notarize_artifacts = 7
     cas_source_authenticate = 8
+
+
+class TestTaskStatusEnum(IntEnum):
+    created = 1
+    started = 2
+    completed = 3
+    failed = 4
+
+
+class TestStepEnum(IntEnum):
+    install_package = 0
+    stop_enviroment = 1
+    initial_provision = 2
+    start_environment = 3
+    uninstall_package = 4
+    initialize_terraform = 5
+    package_integrity_tests = 6
+    stop_environment = 7

build_analytics/build_analytics/db.py

@@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
 from .models.build_node_stat_db import BuildNodeStatDB
 from .models.db_config import DbConfig
 from .models.web_node_stat_db import WebNodeStatDB
+from .models.test_task_db import TestTaskDB
 
 
 class DB():
@@ -229,3 +230,28 @@ class DB():
         cur.execute(sql, (build_task_id, stat_name_id))
         val = int(cur.fetchone()[0])
         return val == 1
+
+    def insert_test_task(self, task: TestTaskDB):
+        cur = self.__conn.cursor()
+        # inserting test task itself
+        sql = '''
+                INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
+                VALUES
+                  (%s, %s, %s, %s, %s, %s);
+              '''
+        cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
+                          task.package_fullname, task.started_at))
+
+        if task.steps_stats:
+            # inserting  test steps stats
+            for ss in task.steps_stats:
+                sql = '''
+                    INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
+                    VALUES
+                        (%s, %s, %s, %s);
+                    '''
+                cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
+                            ss.start_ts, ss.finish_ts))
+
+        # commiting changes
+        self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -1,13 +1,13 @@
 # pylint: disable=relative-beyond-top-level
 
 import logging
-from typing import List, Dict
+from typing import Dict, List
 
-from ..models.extractor_config import ExtractorConfig
-from ..const import BuildTaskEnum
-from ..models.build import BuildTask
-from ..db import DB
 from ..api_client import APIclient
+from ..const import BuildTaskEnum
+from ..db import DB
+from ..models.build import BuildTask
+from ..models.extractor_config import ExtractorConfig
 
 
 class Extractor:
@@ -36,7 +36,7 @@ class Extractor:
                     break
 
                 # inserting build build tasks and build tasks statistics
-                logging.info("inserting %s", build.id)
+                logging.info('inserting %s', build.id)
                 try:
                     self.db.insert_build(build.as_db_model())
                 except Exception as error:  # pylint: disable=broad-except
@@ -45,6 +45,8 @@ class Extractor:
                     continue
 
                 for build_task in build.build_tasks:
+                    logging.info('build %s: inserting build task %s',
+                                 build.id, build_task.id)
                     try:
                         self.db.insert_buildtask(build_task.as_db_model(),
                                                  build_task.web_node_stats.as_db_model(
@@ -52,8 +54,17 @@ class Extractor:
                                                  build_task.build_node_stats.as_db_model(
                             build_task.id))
                     except Exception as error:  # pylint: disable=broad-except
-                        logging.error('failed to insert build task %d: %s',
-                                      build_task.id, error, exc_info=True)
+                        logging.error('build %s: failed to insert build task %d: %s',
+                                      build.id, build_task.id, error, exc_info=True)
+
+                    logging.info(
+                        'getting test tasks for build task %s', build_task.id)
+                    test_tasks = self.api.get_test_tasks(build_task.id)
+                    logging.info('received %d tests tasks', len(test_tasks))
+                    for t in test_tasks:
+                        logging.info(
+                            'build task %s: inserting test task %s', build_task.id, t.id)
+                        self.db.insert_test_task(t.as_db_model())
                 build_count += 1
             page_num += 1
         return build_count

build_analytics/build_analytics/models/sign_task_db.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
 
 
 class SignTaskDB(BaseModel):

build_analytics/build_analytics/models/test_step_stat.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+
+class TestStepStat(BaseModel):
+    start_ts: Optional[datetime] = None
+    finish_ts: Optional[datetime] = None

build_analytics/build_analytics/models/test_step_stat_db.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+from typing import Optional
+
+
+class TestStepStatDB(BaseModel):
+    test_task_id: int
+    stat_name_id: int
+    start_ts: Optional[float] = None
+    finish_ts: Optional[float] = None

build_analytics/build_analytics/models/test_steps_stats.py

@@ -0,0 +1,36 @@
+from typing import List, Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from ..const import TestStepEnum
+from .test_step_stat import TestStepStat
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestStepsStats(BaseModel):
+    install_package: Optional[TestStepStat] = None
+    stop_environment: Optional[TestStepStat] = None
+    initial_provision: Optional[TestStepStat] = None
+    start_environment: Optional[TestStepStat] = None
+    uninstall_package: Optional[TestStepStat] = None
+    initialize_terraform: Optional[TestStepStat] = None
+    package_integrity_tests: Optional[TestStepStat] = None
+
+    def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
+        result = []
+        for field_name in self.__fields__.keys():
+            stats: TestStepStat = getattr(self, field_name)
+            if not stats:
+                continue
+            start_ts = stats.start_ts.timestamp() \
+                if stats.start_ts else None
+            finish_ts = stats.finish_ts.timestamp() \
+                if stats.finish_ts else None
+            stat_name_id = TestStepEnum[field_name].value
+
+            test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
+                                               stat_name_id=stat_name_id,
+                                               start_ts=start_ts,
+                                               finish_ts=finish_ts)
+            result.append(test_step_stat_db)
+        return result

build_analytics/build_analytics/models/test_task.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_task_db import TestTaskDB
+from .test_steps_stats import TestStepsStats
+
+
+class TestTask(BaseModel):
+    id: int
+    build_task_id: int
+    revision: int
+    status: int
+    package_fullname: str
+    started_at: Optional[datetime] = None
+    steps_stats: Optional[TestStepsStats] = None
+
+    def as_db_model(self) -> TestTaskDB:
+        started_at = self.started_at.timestamp() \
+            if self.started_at else None
+        params = {
+            'id': self.id,
+            'build_task_id': self.build_task_id,
+            'revision': self.revision,
+            'status_id': self.status,
+            'package_fullname': self.package_fullname,
+            'started_at': started_at,
+            'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
+        }
+        return TestTaskDB(**params)

build_analytics/build_analytics/models/test_task_db.py

@@ -0,0 +1,17 @@
+from typing import List, Optional
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestTaskDB(BaseModel):
+    """
+    Test task as it received from/sent to database
+    """
+    id: int
+    build_task_id: int
+    revision: int
+    status_id: int
+    package_fullname: str
+    started_at: Optional[float] = None
+    steps_stats: Optional[List[TestStepStatDB]] = None

build_analytics/migrations/1.sql

@@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS builds_finished_at
 ON builds(finished_at);
 
 
--- build_taks_enum
+-- build_tasks_enum
 CREATE TABLE IF NOT EXISTS build_task_status_enum(
     id INTEGER PRIMARY KEY,
     value VARCHAR(15)
@@ -53,7 +53,7 @@ CREATE TABLE web_node_stats_enum (
 );
 
 INSERT INTO web_node_stats_enum (id, value)
-VALUEs
+VALUES
     (0, 'build_done'),
     (1, 'logs_processing'),
     (2, 'packages_processing');

build_analytics/migrations/3.sql

@@ -0,0 +1,77 @@
+BEGIN;
+
+-- test_tasks_status_enum
+CREATE TABLE  test_tasks_status_enum(
+    id INTEGER PRIMARY KEY,
+    value VARCHAR(15)
+);
+
+INSERT INTO test_tasks_status_enum (id, value)
+VALUES
+    (1, 'created'),
+    (2, 'started'),
+    (3, 'completed'),
+    (4, 'failed');
+
+
+-- test_tasks
+CREATE TABLE test_tasks (
+    id INTEGER PRIMARY KEY,
+    build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
+    revision INTEGER,
+    status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
+    package_fullname VARCHAR(100),
+    started_at DOUBLE PRECISION
+);
+
+CREATE INDEX test_tasks_build_task_id
+ON test_tasks(build_task_id);
+
+CREATE INDEX test_tasks_build_status_id
+ON test_tasks(status_id);
+
+CREATE INDEX test_tasks_package_fullname
+ON test_tasks(package_fullname);
+
+
+-- test_steps_enum
+CREATE TABLE test_steps_enum (
+    id INTEGER PRIMARY KEY,
+    value VARCHAR(50)
+);
+
+INSERT INTO test_steps_enum (id, value)
+VALUES
+    (0, 'install_package'),
+    (1, 'stop_environment'),
+    (2, 'initial_provision'),
+    (3, 'start_environment'),
+    (4, 'uninstall_package'),
+    (5, 'initialize_terraform'),
+    (6, 'package_integrity_tests'),
+    (7, 'stop_environment');
+
+
+
+-- test_steps
+CREATE TABLE test_steps_stats(
+    test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
+    stat_name_id  INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
+    start_ts DOUBLE PRECISION,
+    finish_ts DOUBLE PRECISION
+);
+
+ALTER TABLE test_steps_stats
+ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
+
+CREATE INDEX test_steps_stats_start_ts
+ON test_steps_stats(start_ts);
+
+CREATE INDEX test_steps_stats_end_ts
+ON test_steps_stats(end_ts);
+
+
+UPDATE schema_version
+SET version = 3;
+
+COMMIT;

build_analytics/releases.txt

@@ -7,4 +7,7 @@ First version
 - Added metrics for build steps
 
 0.2.1 (2023-03-15)
- - Added canceled Build task status
+ - Added canceled Build task status
+
+0.3.0 (IN PROGRESS)
+ - Added test tasks stats