added api and db functions

build_analytics/build_analytics/api_client.py

@@ -1,16 +1,19 @@
 from datetime import datetime
 import logging
 from urllib.parse import urljoin
-from typing import Dict, List
+from typing import Dict, List, Any
 
 import requests
 
+
 from .models.build import Build
 from .models.build_task import BuildTask
 from .models.build_node_stats import BuildNodeStats
 from .models.build_stat import BuildStat
 from .models.web_node_stats import WebNodeStats
-
+from .models.test_task import TestTask
+from .models.test_steps_stats import TestStepsStats
+from .models.test_step_stat import TestStepStat
 
 TZ_OFFSET = '+00:00'
 
@@ -138,3 +141,53 @@ class APIclient():
             'build_tasks': build_tasks}
 
         return Build(**params)
+
+    def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
+        ep = f'/api/v1/tests/{build_task_id}/latest'
+        url = urljoin(self.api_root, ep)
+        headers = {'accept': 'application/json'}
+
+        response = requests.get(
+            url, headers=headers, timeout=self.timeout)
+        response.raise_for_status()
+        return self.__parse_test_tasks()
+
+    def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
+                           build_task_id: int,
+                           started_at: str = None) -> List[TestTask]:
+        result: List[TestTask] = []
+        for task in raw_tasks:
+            if task['alts_response']:
+                started_raw = task['alts_response']['stats']['started_at']
+                started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
+                stats_raw = task['alts_response']['stats']
+                results_raw = task['results']['tests']
+                step_stats = self.__parse_test_steps_stats(
+                    stats_raw, results_raw)
+            else:
+                started_at = None
+                step_stats = None
+            params = {
+                'id': task['id'],
+                'build_task_id': build_task_id,
+                'revision': task['revision'],
+                'status': task['status'],
+                'package_fullname': '_'.join([raw_tasks['package_name'],
+                                              raw_tasks['package_version'],
+                                              raw_tasks['package_release']]),
+                'started_at': started_at,
+                'step_stats': step_stats
+            }
+            result.append(TestTask(**params))
+        return result
+
+    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats:
+        teast_steps_params = {}
+        for field_name in TestStepsStats.__fields__.keys():
+            try:
+                p = stats_raw[field_name]
+            except KeyError:
+                continue
+            p['success'] = results_raw[field_name]['success']
+            teast_steps_params[field_name] = TestStepStat(**p)
+        return TestStepsStats(**teast_steps_params)

build_analytics/build_analytics/const.py

@@ -3,7 +3,7 @@
 from enum import IntEnum
 
 # supported schema version
-DB_SCHEMA_VER = 2
+DB_SCHEMA_VER = 3
 
 
 # ENUMS
@@ -41,3 +41,20 @@ class BuildNodeStatsEnum(IntEnum):
     build_node_task = 6
     cas_notarize_artifacts = 7
     cas_source_authenticate = 8
+
+
+class TestTaskStatusEnum(IntEnum):
+    created = 1
+    started = 2
+    completed = 3
+    failed = 4
+
+
+class TestStepEnum(IntEnum):
+    install_package = 0
+    stop_enviroment = 1
+    initial_provision = 2
+    start_enviroment = 3
+    uninstall_package = 4
+    initialize_terraform = 5
+    package_integrity_tests = 6

build_analytics/build_analytics/db.py

@@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
 from .models.build_node_stat_db import BuildNodeStatDB
 from .models.db_config import DbConfig
 from .models.web_node_stat_db import WebNodeStatDB
+from .models.test_task_db import TestTaskDB
 
 
 class DB():
@@ -229,3 +230,26 @@ class DB():
         cur.execute(sql, (build_task_id, stat_name_id))
         val = int(cur.fetchone()[0])
         return val == 1
+
+    def insert_test_task(self, task: TestTaskDB):
+        # inserting test task itself
+        sql = '''
+                INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
+                VALUES (%s, %s, %s, %s, %s, %s);
+              '''
+
+        cur = self.__conn.cursor()
+        cur.execute(sql, (task.id, task.build_task_id, task.status_id,
+                          task.package_fullname, task.started_at))
+        # inserting  test steps stats
+        for ss in task.steps_stats:
+            sql = '''
+                  INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
+                  VALUES
+                    (%s, %s, %s, %s, %s);
+                  '''
+            cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
+                        ss.start_ts, ss.end_ts, ss.success))
+
+        # commiting changes
+        self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -1,13 +1,13 @@
 # pylint: disable=relative-beyond-top-level
 
 import logging
-from typing import List, Dict
+from typing import Dict, List
 
-from ..models.extractor_config import ExtractorConfig
-from ..const import BuildTaskEnum
-from ..models.build import BuildTask
-from ..db import DB
 from ..api_client import APIclient
+from ..const import BuildTaskEnum
+from ..db import DB
+from ..models.build import BuildTask
+from ..models.extractor_config import ExtractorConfig
 
 
 class Extractor:
@@ -36,7 +36,7 @@ class Extractor:
                     break
 
                 # inserting build build tasks and build tasks statistics
-                logging.info("inserting %s", build.id)
+                logging.info('inserting %s', build.id)
                 try:
                     self.db.insert_build(build.as_db_model())
                 except Exception as error:  # pylint: disable=broad-except
@@ -45,6 +45,8 @@ class Extractor:
                     continue
 
                 for build_task in build.build_tasks:
+                    logging.info('build %s: inserting build task %s',
+                                 build.id, build_task.id)
                     try:
                         self.db.insert_buildtask(build_task.as_db_model(),
                                                  build_task.web_node_stats.as_db_model(
@@ -52,8 +54,8 @@ class Extractor:
                                                  build_task.build_node_stats.as_db_model(
                             build_task.id))
                     except Exception as error:  # pylint: disable=broad-except
-                        logging.error('failed to insert build task %d: %s',
-                                      build_task.id, error, exc_info=True)
+                        logging.error('build %s: failed to insert build task %d: %s',
+                                      build.id, build_task.id, error, exc_info=True)
                 build_count += 1
             page_num += 1
         return build_count

build_analytics/build_analytics/models/test_step_stat.py

@@ -0,0 +1,10 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+
+class TestStepStat(BaseModel):
+    start_ts: Optional[datetime] = None
+    end_ts: Optional[datetime] = None
+    success: bool

build_analytics/build_analytics/models/test_step_stat_db.py

@@ -0,0 +1,10 @@
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+from typing import Optional
+
+
+class TestStepStatDB(BaseModel):
+    test_task_id: int
+    stat_name_id: int
+    start_ts: Optional[float] = None
+    end_ts: Optional[float] = None
+    success: bool

build_analytics/build_analytics/models/test_steps_stats.py

@@ -0,0 +1,37 @@
+from typing import List, Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from ..const import TestStepEnum
+from .test_step_stat import TestStepStat
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestStepsStats(BaseModel):
+    install_package: Optional[TestStepStat] = None
+    stop_environment: Optional[TestStepStat] = None
+    initial_provision: Optional[TestStepStat] = None
+    start_environment: Optional[TestStepStat] = None
+    uninstall_package: Optional[TestStepStat] = None
+    initialize_terraform: Optional[TestStepStat] = None
+    package_integrity_tests: Optional[TestStepStat] = None
+
+    def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
+        result = []
+        for field_name in self.__fields__.keys():
+            stats: TestStepStat = getattr(self, field_name)
+            if not stats:
+                continue
+            start_ts = stats.start_ts.timestamp() \
+                if stats.start_ts else None
+            end_ts = stats.end_ts.timestamp() \
+                if stats.end_ts else None
+            stat_name_id = TestStepEnum[field_name].value
+
+            test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
+                                               stat_name_id=stat_name_id,
+                                               start_ts=start_ts,
+                                               end_ts=end_ts,
+                                               success=stats.success)
+            result.append(test_step_stat_db)
+        return result

build_analytics/build_analytics/models/test_task.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_task_db import TestTaskDB
+from .test_steps_stats import TestStepsStats
+
+
+class TestTask(BaseModel):
+    id: int
+    build_task_id: int
+    revision: int
+    status: int
+    package_fullname: str
+    started_at: Optional[datetime] = None
+    steps_stats: TestStepsStats
+
+    def as_db_model(self) -> TestTaskDB:
+        started_at = self.started_at.timestamp() \
+            if self.started_at else None
+        params = {
+            'id': self.id,
+            'build_task_id': self.build_task_id,
+            'revision': self.revision,
+            'status': self.status,
+            'package_fullname': self.package_fullname,
+            'started_at': started_at,
+            'steps_stats': self.step_stats.as_db(self.id)
+        }
+        return TestTaskDB(**params)

build_analytics/build_analytics/models/test_task_db.py

@@ -0,0 +1,17 @@
+from typing import List, Optional
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestTaskDB(BaseModel):
+    """
+    Test task as it received from/sent to database
+    """
+    id: int
+    build_task_id: int
+    revision: int
+    status_id: int
+    package_fullname: str
+    started_at: float
+    steps_stats: List[TestStepStatDB] = None

build_analytics/migrations/3.sql

@@ -1,75 +1,74 @@
 BEGIN;
 
--- test_task_status_enum
-CREATE TABLE  test_task_status_enum(
+-- test_tasks_status_enum
+CREATE TABLE  test_tasks_status_enum(
     id INTEGER PRIMARY KEY,
     value VARCHAR(15)
 );
 
-
-
-
-INSERT INTO test_task_status_enum (id, value)
+INSERT INTO test_tasks_status_enum (id, value)
 VALUES
-    (0, 'created'),
-    (1, 'started'),
-    (2, 'completed'),
-    (3, 'failed');
+    (1, 'created'),
+    (2, 'started'),
+    (3, 'completed'),
+    (4, 'failed');
 
 
--- test_task
-CREATE TABLE test_task (
+-- test_tasks
+CREATE TABLE test_tasks (
     id INTEGER PRIMARY KEY,
     build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
     revision INTEGER,
-    status_id INTEGER REFERENCES test_task_status_enum(id) ON DELETE SET NULL,
-    package_fullname VARCHAR(100)
+    status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
+    package_fullname VARCHAR(100),
+    started_at DOUBLE PRECISION
 );
 
-CREATE INDEX test_task_build_task_id
-ON test_task(build_task_id);
+CREATE INDEX test_tasks_build_task_id
+ON test_tasks(build_task_id);
 
-CREATE INDEX test_task_build_status_id
-ON test_task(status_id);
+CREATE INDEX test_tasks_build_status_id
+ON test_tasks(status_id);
 
-CREATE INDEX test_task_package_fullname
-ON test_task(package_fullname);
+CREATE INDEX test_tasks_package_fullname
+ON test_tasks(package_fullname);
 
 
--- test_step_enum
-CREATE TABLE test_step_enum (
+-- test_steps_enum
+CREATE TABLE test_steps_enum (
     id INTEGER PRIMARY KEY,
     value VARCHAR(50)
 );
 
-INSERT INTO test_step_enum (id, value)
+INSERT INTO test_steps_enum (id, value)
 VALUES
-    (1, 'install_package'),
-    (2, 'stop_environment'),
-    (3, 'initial_provision'),
-    (4, 'start_environment'),
-    (5, 'uninstall_package'),
-    (6, 'initialize_terraform'),
-    (7, 'package_integrity_tests');
+    (0, 'install_package'),
+    (1, 'stop_environment'),
+    (2, 'initial_provision'),
+    (3, 'start_environment'),
+    (4, 'uninstall_package'),
+    (5, 'initialize_terraform'),
+    (6, 'package_integrity_tests');
 
 
 
--- test_step
-CREATE TABLE test_step_stats(
+-- test_steps
+CREATE TABLE test_steps_stats(
     test_task_id INTEGER,
-    stat_name_id  INTEGER REFERENCES test_step_enum(id) ON DELETE SET NULL,
+    stat_name_id  INTEGER REFERENCES (id) ON DELETE SET NULL,
     start_ts DOUBLE PRECISION,
-    end_ts DOUBLE PRECISION
+    end_ts DOUBLE PRECISION,
+    success BOOLEAN
 );
 
-ALTER TABLE test_step_stats
-ADD CONSTRAINT test_step_stats_unique UNIQUE (test_task_id, stat_name_id);
+ALTER TABLE test_steps_stats
+ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
 
-CREATE INDEX test_step_stats_start_ts
-ON test_step_stats(start_ts);
+CREATE INDEX test_steps_stats_start_ts
+ON test_steps_stats(start_ts);
 
-CREATE INDEX test_step_stats_end_ts
-ON test_step_stats(end_ts);
+CREATE INDEX test_steps_stats_end_ts
+ON test_steps_stats(end_ts);
 
 
 UPDATE schema_version