Release 0.3.5 (2023-06-01)

build_analytics:
  ALBS-1103 start using persistent HTTP connections

build_analytics/build_analytics/api_client.py

@@ -1,16 +1,19 @@
 from datetime import datetime
 import logging
 from urllib.parse import urljoin
-from typing import Dict, List
+from typing import Dict, List, Any, Optional
 
 import requests
 
+
 from .models.build import Build
 from .models.build_task import BuildTask
 from .models.build_node_stats import BuildNodeStats
 from .models.build_stat import BuildStat
 from .models.web_node_stats import WebNodeStats
-
+from .models.test_task import TestTask
+from .models.test_steps_stats import TestStepsStats
+from .models.test_step_stat import TestStepStat
 
 TZ_OFFSET = '+00:00'
 
@@ -24,6 +27,8 @@ class APIclient():
         self.api_root = api_root
         self.jwt = jwt
         self.timeout = timeout
+        # will be set at first call of __send_request
+        self.session: Optional[requests.Session] = None
 
     def get_builds(self, page_num: int = 1) -> List[Build]:
         ep = '/api/v1/builds'
@@ -31,8 +36,7 @@ class APIclient():
         params = {'pageNumber': page_num}
         headers = {'accept': 'appilication/json'}
 
-        response = requests.get(
+        response = self.__send_request(url, 'get', params, headers)
-            url, params=params, headers=headers, timeout=self.timeout)
         response.raise_for_status()
 
         result = []
@@ -44,11 +48,18 @@ class APIclient():
                               b, err, exc_info=True)
         return result
 
-    def get_build(self, build_id: int) -> Build:
+    def get_build(self, build_id: int) -> Optional[Build]:
+        '''
+        method returns None if build was deleted from ALBS
+        '''
         ep = f'/api/v1/builds/{build_id}'
         url = urljoin(self.api_root, ep)
         headers = {'accept': 'application/json'}
-        response = requests.get(url, headers=headers, timeout=self.timeout)
+        response = self.__send_request(url, 'get', headers=headers)
+
+        if response.status_code == 404:
+            return None
+
         response.raise_for_status()
         return self._parse_build(response.json())
 
@@ -101,7 +112,8 @@ class APIclient():
                 if not task['performance_stats']:
                     logging.warning(
                         "no perfomance_stats for build_id: %s, build_task_id: %s", build_id, task['id'])
-                    stats = {'build_node_stats': {}, 'build_done_stats': {}}
+                    stats: dict[str, Any] = {
+                        'build_node_stats': {}, 'build_done_stats': {}}
                 else:
                     stats = task['performance_stats'][0]['statistics']
 
@@ -138,3 +150,110 @@ class APIclient():
             'build_tasks': build_tasks}
 
         return Build(**params)
+
+    def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
+        result: List[TestTask] = []
+        revision = 1
+        while True:
+            ep = f'/api/v1/tests/{build_task_id}/{revision}'
+            url = urljoin(self.api_root, ep)
+            headers = {'accept': 'application/json'}
+
+            response = requests.get(
+                url, headers=headers, timeout=self.timeout)
+            response.raise_for_status()
+            raw_tasks = response.json()
+            if len(raw_tasks) == 0:
+                break
+            result = result + self.__parse_test_tasks(raw_tasks, build_task_id)
+            revision += 1
+        return result
+
+    def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
+                           build_task_id: int) -> List[TestTask]:
+        result: List[TestTask] = []
+        for task in raw_tasks:
+            if task['alts_response']:
+                try:
+                    stats_raw = task['alts_response']['stats']
+                except KeyError:
+                    steps_stats = None
+                else:
+                    steps_stats = self.__parse_test_steps_stats(stats_raw)
+            else:
+                steps_stats = None
+            params = {
+                'id': task['id'],
+                'build_task_id': build_task_id,
+                'revision': task['revision'],
+                'status': task['status'],
+                'package_fullname': '_'.join([task['package_name'],
+                                              task['package_version'],
+                                              task['package_release']]),
+                'started_at': self.__get_test_task_started_at(
+                    steps_stats) if steps_stats else None,
+                'steps_stats': steps_stats
+            }
+
+            result.append(TestTask(**params))
+        return result
+
+    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
+        teast_steps_params = {}
+        for field_name in TestStepsStats.__fields__.keys():
+            try:
+                p = stats_raw[field_name]
+            except KeyError:
+                continue
+            # there are must be a better way
+            for k in ['start_ts', 'finish_ts']:
+                if k in p:
+                    p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
+            teast_steps_params[field_name] = TestStepStat(**p)
+        return TestStepsStats(**teast_steps_params)
+
+    def __get_test_task_started_at(self, stats: TestStepsStats) -> Optional[datetime]:
+        """
+        getting started_at attribute for test by using oldest start_ts timestamp
+        among all test tasks steps
+        """
+        if not stats:
+            return None
+
+        start_ts = None
+        for field_name in stats.__fields__.keys():
+            stat: TestStepStat = getattr(stats, field_name)
+            if not stat:
+                continue
+            if not start_ts or start_ts > stat.start_ts:
+                start_ts = stat.start_ts
+
+        return start_ts
+
+    def __send_request(self,
+                       url: str,
+                       method: str,
+                       params: Optional[Dict[str, Any]] = None,
+                       headers: Optional[Dict[str, Any]] = None,
+                       ) -> requests.Response:
+        """
+        Simple wrapper around requests.get/posts.. methods 
+        so we can use same session between API calls
+        """
+        if not self.session:
+            self.session = requests.Session()
+
+        m = getattr(self.session, method, None)
+        if not m:
+            raise ValueError(f"method {method} is not supported")
+
+        # pylint: disable=not-callable
+        return m(url, params=params, headers=headers, timeout=self.timeout)
+
+    def close_session(self):
+        if self.session:
+            self.session.close()
+            self.session = None
+
+    def __del__(self):
+        self.close_session()

build_analytics/build_analytics/const.py

@@ -3,7 +3,7 @@
 from enum import IntEnum
 
 # supported schema version
-DB_SCHEMA_VER = 2
+DB_SCHEMA_VER = 3
 
 
 # ENUMS
@@ -41,3 +41,21 @@ class BuildNodeStatsEnum(IntEnum):
     build_node_task = 6
     cas_notarize_artifacts = 7
     cas_source_authenticate = 8
+
+
+class TestTaskStatusEnum(IntEnum):
+    created = 1
+    started = 2
+    completed = 3
+    failed = 4
+
+
+class TestStepEnum(IntEnum):
+    install_package = 0
+    stop_enviroment = 1
+    initial_provision = 2
+    start_environment = 3
+    uninstall_package = 4
+    initialize_terraform = 5
+    package_integrity_tests = 6
+    stop_environment = 7

build_analytics/build_analytics/db.py

@@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
 from .models.build_node_stat_db import BuildNodeStatDB
 from .models.db_config import DbConfig
 from .models.web_node_stat_db import WebNodeStatDB
+from .models.test_task_db import TestTaskDB
 
 
 class DB():
@@ -25,6 +26,18 @@ class DB():
     def __del__(self):
         self.close_conn()
 
+    def row_exists(self, pk: int, table: str) -> bool:
+        assert table in ['builds', 'test_tasks']
+        sql = f'''
+              SELECT COUNT(id)
+              FROM {table}
+              WHERE id = %s;
+              '''
+        cur = self.__conn.cursor()
+        cur.execute(sql, (pk,))
+        val = int(cur.fetchone()[0])
+        return val == 1
+
     def insert_build(self, build: BuildDB):
         sql = '''
                 INSERT INTO builds(id, url, created_at, finished_at)
@@ -49,34 +62,34 @@ class DB():
                           build_task.started_at, build_task.finished_at, build_task.status_id))
 
         # inserting web node stats
-        for stat in web_node_stats:
+        for wn_stat in web_node_stats:
 
             # do not insert empty stats
-            if stat.start_ts is None:
+            if wn_stat.start_ts is None:
                 continue
 
             sql = '''
                     INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
                 '''
-            cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
+            cur.execute(sql, (wn_stat.build_task_id, wn_stat.stat_name_id,
-                        stat.start_ts, stat.end_ts))
+                        wn_stat.start_ts, wn_stat.end_ts))
             logging.debug('raw SQL query: %s', cur.query)
             self.__conn.commit()
 
         # inserting build node stats
-        for stat in build_node_stats:
+        for bn_stat in build_node_stats:
 
             # do not insert empty stats
-            if stat.start_ts is None:
+            if bn_stat.start_ts is None:
                 continue
 
             sql = '''
                     INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
                 '''
-            cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
+            cur.execute(sql, (bn_stat.build_task_id, bn_stat.stat_name_id,
-                        stat.start_ts, stat.end_ts))
+                        bn_stat.start_ts, bn_stat.end_ts))
             logging.debug('raw SQL query: %s', cur.query)
 
         # commiting changes
@@ -99,7 +112,7 @@ class DB():
         self.__conn.commit()
         return cur.rowcount
 
-    def get_unfinished_builds(self) -> Dict[int, Dict[int, int]]:
+    def get_unfinished_builds(self, not_before: datetime) -> Dict[int, Dict[int, int]]:
         """
         Getting list of unfinished builds and build_tasks
         Dict[build_id, Dict[build_task_id, task_status_id]]
@@ -107,11 +120,13 @@ class DB():
         res: Dict[int, Dict[int, int]] = {}
 
         # getting unfinished builds
-        sql = 'SELECT id FROM builds where finished_at is NULL;'
+        sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
+        builds_to_check: Dict[int, bool] = {}
         cur = self.__conn.cursor()
-        cur.execute(sql)
+        cur.execute(sql, (not_before.timestamp(),))
+        logging.debug('raw SQL query: %s', cur.query)
         for row in cur.fetchall():
-            res[row[0]] = {}
+            builds_to_check[row[0]] = True
 
         # getting list of unfinished tasks
         sql = 'SELECT id, build_id, status_id FROM build_tasks WHERE status_id < 2;'
@@ -121,6 +136,8 @@ class DB():
             build_task_id: int = row[0]
             build_id: int = row[1]
             status_id: int = row[2]
+            if build_id not in builds_to_check:
+                continue
             try:
                 res[build_id][build_task_id] = status_id
             except KeyError:
@@ -159,9 +176,10 @@ class DB():
         for stat in web_node_stats:
             logging.debug(
                 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
-            if self.stat_exists(build_task_id=stat.build_task_id,
+            if self.stat_exists(task_id=stat.build_task_id,
                                 stat_name_id=stat.stat_name_id,
-                                table_name='web_node_stats'):
+                                table_name='web_node_stats',
+                                column_name='build_task_id'):
                 sql = '''
                       UPDATE web_node_stats
                       SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@@ -180,12 +198,13 @@ class DB():
             logging.debug('raw SQL query: %s', cur.query)
 
         # updating build_node_stats
-        for stat in build_node_stats:
+        for bn_stat in build_node_stats:
             logging.debug(
-                'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
+                'updating build_node_stats %s build_task %s', bn_stat.stat_name_id, build_task.id)
-            if self.stat_exists(build_task_id=stat.build_task_id,
+            if self.stat_exists(task_id=bn_stat.build_task_id,
-                                stat_name_id=stat.stat_name_id,
+                                stat_name_id=bn_stat.stat_name_id,
-                                table_name='build_node_stats'):
+                                table_name='build_node_stats',
+                                column_name='build_task_id'):
                 sql = '''
                       UPDATE build_node_stats
                       SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@@ -197,9 +216,9 @@ class DB():
                         VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
                       '''
             params = {'build_task_id': build_task.id,
-                      'stat_name_id': stat.stat_name_id,
+                      'stat_name_id': bn_stat.stat_name_id,
-                      'start_ts': stat.start_ts,
+                      'start_ts': bn_stat.start_ts,
-                      'end_ts': stat.end_ts}
+                      'end_ts': bn_stat.end_ts}
             logging.debug('raw SQL query: %s', cur.query)
             cur.execute(sql, params)
 
@@ -219,13 +238,94 @@ class DB():
             return None
         return int(val[0])
 
-    def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool:
+    def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool:
         sql = f'''
-                SELECT COUNT(build_task_id)
+                SELECT COUNT({column_name})
                 FROM {table_name}
-                WHERE build_task_id = %s AND stat_name_id = %s;
+                WHERE {column_name} = %s AND stat_name_id = %s;
              '''
         cur = self.__conn.cursor()
-        cur.execute(sql, (build_task_id, stat_name_id))
+        cur.execute(sql, (task_id, stat_name_id))
         val = int(cur.fetchone()[0])
         return val == 1
+
+    def get_build_tasks_for_tests_update(self, not_before: datetime) -> List[int]:
+        '''
+        Getting build tasks id for test tasks that we need to update
+        https://cloudlinux.atlassian.net/browse/ALBS-1060
+        '''
+        cur = self.__conn.cursor()
+        sql = '''
+              SELECT bt.id
+              FROM build_tasks AS bt
+              INNER JOIN builds AS b
+                ON b.id = bt.build_id
+              WHERE b.created_at > %s;
+              '''
+        cur.execute(sql, (not_before.timestamp(),))
+        logging.debug('raw SQL query: %s', cur.query)
+        result = [int(row[0]) for row in cur.fetchall()]
+        return result
+
+    def insert_update_test_tasks(self, test_tasks: List[TestTaskDB]):
+        cur = self.__conn.cursor()
+        # test tasks
+        for task in test_tasks:
+            if self.row_exists(pk=task.id, table='test_tasks'):
+                sql = '''
+                    UPDATE test_tasks
+                    SET revision = %s,
+                        status_id = %s,
+                        started_at = %s
+                    WHERE id = %s;
+                '''
+                cur.execute(sql, (task.revision, task.status_id,
+                            task.started_at, task.id))
+                assert cur.rowcount == 1
+            else:
+                sql = '''
+                      INSERT INTO test_tasks(
+                        id, build_task_id, revision, status_id, package_fullname, started_at)
+                      VALUES
+                      (%s, %s, %s, %s, %s, %s);
+                      '''
+                cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
+                                  task.package_fullname, task.started_at))
+
+            # test step
+            if not task.steps_stats:
+                continue
+            for s in task.steps_stats:
+                logging.debug('test_task_id %s, stat_name_id %s',
+                              s.test_task_id, s.stat_name_id)
+                if self.stat_exists(s.test_task_id,
+                                    s.stat_name_id,
+                                    'test_steps_stats',
+                                    'test_task_id'):
+                    sql = '''
+                            UPDATE test_steps_stats
+                            SET start_ts = %s,
+                                finish_ts = %s
+                            WHERE test_task_id = %s AND stat_name_id = %s;
+                          '''
+                    cur.execute(sql, (s.start_ts, s.finish_ts,
+                                s.test_task_id, s.stat_name_id))
+                    assert cur.rowcount == 1
+                else:
+                    sql = '''
+                          INSERT INTO test_steps_stats (
+                            test_task_id, stat_name_id, start_ts, finish_ts)
+                          VALUES (%s, %s, %s, %s);
+                          '''
+                    cur.execute(sql, (s.test_task_id, s.stat_name_id,
+                                s.start_ts, s.finish_ts))
+        # commiting changes
+        self.__conn.commit()
+
+    def delete_build(self, build_id: int):
+        params = (build_id,)
+        sql = "DELETE FROM builds WHERE id = %s;"
+        cur = self.__conn.cursor()
+
+        cur.execute(sql, params)
+        self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -1,19 +1,20 @@
 # pylint: disable=relative-beyond-top-level
 
+from datetime import datetime, timedelta
 import logging
-from typing import List, Dict
+from typing import Dict, List
+
 
-from ..models.extractor_config import ExtractorConfig
-from ..const import BuildTaskEnum
-from ..models.build import BuildTask
-from ..db import DB
 from ..api_client import APIclient
+from ..const import BuildTaskEnum
+from ..db import DB
+from ..models.build import BuildTask
+from ..models.extractor_config import ExtractorConfig
 
 
 class Extractor:
     def __init__(self, config: ExtractorConfig, api: APIclient, db: DB):
-        self.start_from = config.start_from
+        self.config = config
-        self.oldest_build_age = config.oldest_build_age
         self.api = api
         self.db = db
 
@@ -22,21 +23,27 @@ class Extractor:
         page_num = 1
         last_build_id = self.db.get_latest_build_id()
         if not last_build_id:
-            last_build_id = self.start_from - 1
+            last_build_id = self.config.start_from - 1
         logging.info("last_build_id: %s", last_build_id)
         stop = False
 
         while not stop:
+            oldest_build_age = datetime.now().astimezone() - \
+                timedelta(days=self.config.data_store_days)
             logging.info("page: %s", page_num)
             for build in self.api.get_builds(page_num):
                 # check if we shoud stop processing build
                 if build.id <= last_build_id or \
-                   build.created_at <= self.oldest_build_age:
+                   build.created_at <= oldest_build_age:
                     stop = True
                     break
 
+                # some builds could move from one page to another
+                if self.db.row_exists(pk=build.id, table='builds'):
+                    continue
+
                 # inserting build build tasks and build tasks statistics
-                logging.info("inserting %s", build.id)
+                logging.info('inserting %s', build.id)
                 try:
                     self.db.insert_build(build.as_db_model())
                 except Exception as error:  # pylint: disable=broad-except
@@ -45,6 +52,8 @@ class Extractor:
                     continue
 
                 for build_task in build.build_tasks:
+                    logging.info('build %s: inserting build task %s',
+                                 build.id, build_task.id)
                     try:
                         self.db.insert_buildtask(build_task.as_db_model(),
                                                  build_task.web_node_stats.as_db_model(
@@ -52,16 +61,26 @@ class Extractor:
                                                  build_task.build_node_stats.as_db_model(
                             build_task.id))
                     except Exception as error:  # pylint: disable=broad-except
-                        logging.error('failed to insert build task %d: %s',
+                        logging.error('build %s: failed to insert build task %d: %s',
-                                      build_task.id, error, exc_info=True)
+                                      build.id, build_task.id, error, exc_info=True)
+
+                    logging.info(
+                        'getting test tasks for build task %s', build_task.id)
+                    test_tasks = self.api.get_test_tasks(build_task.id)
+                    logging.info('received %d tests tasks', len(test_tasks))
+                    if len(test_tasks) > 0:
+                        logging.info('inserting test tasks')
+                        as_db = [t.as_db_model() for t in test_tasks]
+                        self.db.insert_update_test_tasks(as_db)
                 build_count += 1
             page_num += 1
         return build_count
 
     def build_cleanup(self):
-        logging.info('Removing all buidls older then %s',
+        oldest_to_keep = datetime.now().astimezone() - \
-                     self.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
+            timedelta(days=self.config.data_store_days)
-        removed_count = self.db.cleanup_builds(self.oldest_build_age)
+        logging.info('Removing all buidls older then %s', oldest_to_keep)
+        removed_count = self.db.cleanup_builds(oldest_to_keep)
         logging.info('removed %d entries', removed_count)
 
     def __update_build_tasks(self, build_tasks: List[BuildTask],
@@ -91,12 +110,20 @@ class Extractor:
                     b.build_id, b.id, BuildTaskEnum(b.status_id).name)
 
     def update_builds(self):
-        logging.info('Getting list of tasks from DB')
+        not_before = datetime.now().astimezone() - \
-        unfinished_tasks = self.db.get_unfinished_builds()
+            timedelta(days=self.config.oldest_to_update_days)
+        logging.info('Getting unfinished builds that were created after %s ',
+                     not_before)
+        unfinished_tasks = self.db.get_unfinished_builds(not_before)
         for build_id, build_tasks_db in unfinished_tasks.items():
             try:
                 logging.info('Getting status of build %d', build_id)
                 build = self.api.get_build(build_id)
+                if not build:
+                    logging.warning(
+                        "build %s was deleted from albs, removing it", build_id)
+                    self.db.delete_build(build_id)
+                    continue
 
                 logging.info('Updating build tasks')
                 build_tasks_to_check = [
@@ -114,3 +141,22 @@ class Extractor:
             except Exception as err:  # pylint: disable=broad-except
                 logging.error("Cant process build %d:  %s",
                               build_id, err, exc_info=True)
+
+    def updating_test_tasks(self):
+        not_before = datetime.now().astimezone() - \
+            timedelta(days=self.config.oldest_to_update_days)
+        logging.info('getting build tasks for builds created after %s',
+                     not_before)
+        build_task_ids = self.db.get_build_tasks_for_tests_update(
+            not_before)
+        for build_task_id in build_task_ids:
+            try:
+                logging.info('getting tests for build task %s', build_task_id)
+                tasks_api = self.api.get_test_tasks(build_task_id)
+                logging.info('updating test tasks')
+                tasks_db = [t.as_db_model() for t in tasks_api]
+                self.db.insert_update_test_tasks(tasks_db)
+            except Exception as err:  # pylint: disable=broad-except
+                logging.error(
+                    'failed to update tests for %d build task: %s',
+                    build_task_id, err, exc_info=True)

build_analytics/build_analytics/extractor/start.py

@@ -1,8 +1,8 @@
-from datetime import datetime, timedelta
 import logging
 from logging.handlers import RotatingFileHandler
 import sys
 import time
+from typing import Dict, Any
 
 import yaml
 
@@ -15,20 +15,6 @@ from ..models.extractor_config import ExtractorConfig
 from ..models.db_config import DbConfig
 
 
-def __get_oldest_build_age(config: dict) -> datetime:
-    oldest_build_age = datetime.now().astimezone() \
-        - timedelta(days=config['data_store_days'])
-    return oldest_build_age
-
-
-def __get_db_config(config: dict) -> DbConfig:
-    return DbConfig(name=config['db_name'],
-                    port=int(config['db_port']),
-                    host=config['db_host'],
-                    username=config['db_username'],
-                    password=config['db_password'])
-
-
 def __get_config(yml_path: str) -> ExtractorConfig:
     """
     get_config loads yml file and generates  instance
@@ -36,9 +22,15 @@ def __get_config(yml_path: str) -> ExtractorConfig:
     with open(yml_path, 'r', encoding='utf-8') as flr:
         raw = yaml.safe_load(flr)
 
-    # adding new attrs
+    # Dbconfig
-    raw['oldest_build_age'] = __get_oldest_build_age(raw)
+    db_params: Dict[str, Any] = {'name': raw['db_name'],
-    raw['db_config'] = __get_db_config(raw)
+                                 'username': raw['db_username'],
+                                 'password': raw['db_password'], }
+    if 'db_port' in raw:
+        db_params['port'] = raw['db_port']
+    if 'db_host' in raw:
+        db_params['host'] = raw['db_host']
+    raw['db_config'] = DbConfig(**db_params)
 
     return ExtractorConfig(**raw)
 
@@ -83,23 +75,34 @@ def start(yml_path: str):
             logging.info(
                 'Build extaction was finished. %d builds were inserted', inserted_count)
 
-        logging.info('Starting old builds removal')
+        logging.info('starting old builds removal')
         try:
             extractor.build_cleanup()
         except Exception as err:  # pylint: disable=broad-except
-            logging.critical("Unhandled exception %s", err, exc_info=True)
+            logging.critical("unhandled exception %s", err, exc_info=True)
         else:
-            logging.info('Cleanup finished')
+            logging.info('cleanup finished')
 
-        logging.info('Updating statuses of unfinished build tasks')
+        logging.info('updating statuses of unfinished build tasks')
         try:
             extractor.update_builds()
         except Exception as err:  # pylint: disable=broad-except
-            logging.critical("Unhandled exception %s", err, exc_info=True)
+            logging.critical("unhandled exception %s", err, exc_info=True)
         else:
-            logging.info('Update finished')
+            logging.info('update finished')
 
+        logging.info('updating/inserting test tasks')
+        try:
+            extractor.updating_test_tasks()
+        except Exception as err:  # pylint: disable=broad-except
+            logging.critical("unhandled exception %s", err, exc_info=True)
+        else:
+            logging.info('test tasks were updated')
+
+        # freeing up resources
         extractor.db.close_conn()
+        extractor.api.close_session()
+
         logging.info("Extraction was finished")
         logging.info("Sleeping for %d seconds", config.scrape_interval)
         time.sleep(config.scrape_interval)

build_analytics/build_analytics/models/db_config.py

@@ -1,9 +1,13 @@
 from pydantic import BaseModel, Field
 
 
+DB_PORT = 5432
+DB_HOST = "localhost"
+
+
 class DbConfig(BaseModel):
     name: str = Field(description="db name")
-    port: int = Field(description="db server port")
+    port: int = Field(description="db server port", default=DB_PORT)
-    host: str = Field(description="db server ip/hostname")
+    host: str = Field(description="db server ip/hostname", default=DB_HOST)
     username: str = Field(description="username to connect with")
     password: str = Field(description="password to connect with1")

build_analytics/build_analytics/models/extractor_config.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 from pathlib import Path
 
 from pydantic import HttpUrl, Field, BaseModel  # pylint: disable=no-name-in-module
@@ -8,9 +7,10 @@ from .db_config import DbConfig
 # DEFAULTS
 ALBS_URL_DEFAULT = 'https://build.almalinux.org'
 LOG_FILE_DEFAULT = '/tmp/extractor.log'
-API_DEFAULT = 30
+API_TIMEOUT_DEFAULT = 30
 SCRAPE_INTERVAL_DEFAULT = 3600
 START_FROM_DEFAULT = 5808
+OLDEST_TO_UPDATE_DAYS_DEFAULT = 7
 
 
 class ExtractorConfig(BaseModel):
@@ -21,14 +21,17 @@ class ExtractorConfig(BaseModel):
                            default=LOG_FILE_DEFAULT)
     albs_url: HttpUrl = Field(description='ALBS root URL',
                               default=ALBS_URL_DEFAULT)
-    oldest_build_age: datetime = \
+    data_store_days: int = \
-        Field(description='oldest build age to store')
+        Field(description='oldest build (in days) to keep in DB')
     jwt: str = Field(description='ALBS JWT token')
     db_config: DbConfig = Field(description="database configuration")
     api_timeout: int = Field(
         description="max time in seconds to wait for API response",
-        default=API_DEFAULT)
+        default=API_TIMEOUT_DEFAULT)
     scrape_interval: int = Field(description='how often (in seconds) we will extract data from ALBS',
                                  default=SCRAPE_INTERVAL_DEFAULT)
     start_from: int = Field(description='build id to start populating empty db with',
                             default=START_FROM_DEFAULT)
+    oldest_to_update_days: int = \
+        Field(description='oldest (in days) unfinished object (build/task/step...) that we will try to update',
+              default=OLDEST_TO_UPDATE_DAYS_DEFAULT)

build_analytics/build_analytics/models/sign_task_db.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
 
 
 class SignTaskDB(BaseModel):

build_analytics/build_analytics/models/test_step_stat.py

@@ -0,0 +1,9 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+
+class TestStepStat(BaseModel):
+    start_ts: Optional[datetime] = None
+    finish_ts: Optional[datetime] = None

build_analytics/build_analytics/models/test_step_stat_db.py

@@ -0,0 +1,9 @@
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+from typing import Optional
+
+
+class TestStepStatDB(BaseModel):
+    test_task_id: int
+    stat_name_id: int
+    start_ts: Optional[float] = None
+    finish_ts: Optional[float] = None

build_analytics/build_analytics/models/test_steps_stats.py

@@ -0,0 +1,36 @@
+from typing import List, Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from ..const import TestStepEnum
+from .test_step_stat import TestStepStat
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestStepsStats(BaseModel):
+    install_package: Optional[TestStepStat] = None
+    stop_environment: Optional[TestStepStat] = None
+    initial_provision: Optional[TestStepStat] = None
+    start_environment: Optional[TestStepStat] = None
+    uninstall_package: Optional[TestStepStat] = None
+    initialize_terraform: Optional[TestStepStat] = None
+    package_integrity_tests: Optional[TestStepStat] = None
+
+    def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
+        result = []
+        for field_name in self.__fields__.keys():
+            stats: TestStepStat = getattr(self, field_name)
+            if not stats:
+                continue
+            start_ts = stats.start_ts.timestamp() \
+                if stats.start_ts else None
+            finish_ts = stats.finish_ts.timestamp() \
+                if stats.finish_ts else None
+            stat_name_id = TestStepEnum[field_name].value
+
+            test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
+                                               stat_name_id=stat_name_id,
+                                               start_ts=start_ts,
+                                               finish_ts=finish_ts)
+            result.append(test_step_stat_db)
+        return result

build_analytics/build_analytics/models/test_task.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_task_db import TestTaskDB
+from .test_steps_stats import TestStepsStats
+
+
+class TestTask(BaseModel):
+    id: int
+    build_task_id: int
+    revision: int
+    status: int
+    package_fullname: str
+    started_at: Optional[datetime] = None
+    steps_stats: Optional[TestStepsStats] = None
+
+    def as_db_model(self) -> TestTaskDB:
+        started_at = self.started_at.timestamp() \
+            if self.started_at else None
+        params = {
+            'id': self.id,
+            'build_task_id': self.build_task_id,
+            'revision': self.revision,
+            'status_id': self.status,
+            'package_fullname': self.package_fullname,
+            'started_at': started_at,
+            'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
+        }
+        return TestTaskDB(**params)

build_analytics/build_analytics/models/test_task_db.py

@@ -0,0 +1,17 @@
+from typing import List, Optional
+from pydantic import BaseModel  # pylint: disable=no-name-in-module
+
+from .test_step_stat_db import TestStepStatDB
+
+
+class TestTaskDB(BaseModel):
+    """
+    Test task as it received from/sent to database
+    """
+    id: int
+    build_task_id: int
+    revision: int
+    status_id: int
+    package_fullname: str
+    started_at: Optional[float] = None
+    steps_stats: Optional[List[TestStepStatDB]] = None

build_analytics/config_default.yml

@@ -10,7 +10,6 @@ albs_url: https://build.almalinux.org
 # required: yes
 jwt: ""
 
-
 # db_host
 # IP/hostname of database server
 # required: no
@@ -28,7 +27,6 @@ db_port: 5432
 # required: yes
 db_username: albs_analytics
 
-
 # db_password
 # password to connect with
 # required: yes
@@ -39,7 +37,6 @@ db_password: super_secret_password
 # required: yes
 db_name: albs_analytics
 
-
 # log_file
 # file to  write logs to
 # required: no
@@ -60,4 +57,16 @@ scrape_interval: 3600
 # build_id to start populating empty db with
 # required: false
 # default: 5808 (first build with correct metrics)
-start_from:
+start_from: 5808
+
+# oldest_to_update_days
+# oldest (in days) unfinished object (build/task/step...) that we will try to update
+# required: false
+# default: 7
+oldest_to_update_days: 7
+
+# api_timeout
+# how long (in seconds) we will wait for API response
+# required: false
+# default: 30
+api_timeout: 30

build_analytics/grafana-dashbords/Build details.json

@@ -87,7 +87,7 @@
       },
       "id": 16,
       "panels": [],
-      "title": "Summary",
+      "title": "Builds",
       "type": "row"
     },
     {
@@ -1261,19 +1261,6 @@
       ],
       "type": "stat"
     },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 12
-      },
-      "id": 23,
-      "panels": [],
-      "title": "Build steps",
-      "type": "row"
-    },
     {
       "datasource": {
         "type": "postgres",
@@ -1423,7 +1410,7 @@
         "h": 8,
         "w": 8,
         "x": 0,
-        "y": 13
+        "y": 12
       },
       "id": 20,
       "options": {
@@ -1677,7 +1664,7 @@
           "format": "table",
           "hide": false,
           "rawQuery": true,
-          "rawSql": "SELECT \n  SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n  ON bs.build_task_id =  build_tasks.id\nWHERE  stat_name_id = 1 AND build_tasks.build_id = $build_id ",
+          "rawSql": "SELECT \n  SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n  ON bs.build_task_id =  build_tasks.id\nWHERE  stat_name_id = 2 AND build_tasks.build_id = $build_id ",
           "refId": "packages_procession",
           "sql": {
             "columns": [
@@ -1735,7 +1722,7 @@
         "h": 8,
         "w": 16,
         "x": 8,
-        "y": 13
+        "y": 12
       },
       "id": 21,
       "options": {
@@ -1981,8 +1968,8 @@
           "format": "table",
           "hide": false,
           "rawQuery": true,
-          "rawSql": "SELECT \n  SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n  ON bs.build_task_id =  build_tasks.id\nWHERE  stat_name_id = 1 AND build_tasks.build_id = $build_id ",
+          "rawSql": "SELECT \n  SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n  ON bs.build_task_id =  build_tasks.id\nWHERE  stat_name_id = 2 AND build_tasks.build_id = $build_id ",
-          "refId": "packages_procession",
+          "refId": "packages_processing",
           "sql": {
             "columns": [
               {
@@ -2005,13 +1992,495 @@
       "title": "Duration (absolute)",
       "type": "bargauge"
     },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 27,
+      "panels": [],
+      "title": "Tests",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 13,
+        "x": 0,
+        "y": 21
+      },
+      "id": 29,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  COUNT(tt.id),\n  enum.value AS status\nFROM test_tasks AS tt\nINNER JOIN test_tasks_status_enum AS enum\n  ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n  ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (status);\n",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
+        }
+      ],
+      "title": "Tests count (group by status)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {}
+        }
+      ],
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 11,
+        "x": 13,
+        "y": 21
+      },
+      "id": 31,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  MAX(t.duration) AS \"MAX\",\n  MIN(t.duration) AS \"MIN\",\n  AVG(t.duration) AS \"AVERAGE\",\n  PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY t.duration) as \"MEDIAN\",\n  PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY t.duration) as \"95TH PERCENTILE\"\nFROM\n  (SELECT \n    tt.started_at * 1000 AS \"started at\",\n    tf.finished_at * 1000 AS \"finished at\", \n    tf.finished_at - tt.started_at AS duration\n  FROM test_tasks AS tt\n  INNER JOIN test_steps_stats AS tss\n    ON tt.id = tss.test_task_id\n  INNER JOIN test_tasks_status_enum AS enum\n    ON tt.status_id = enum.id\n  INNER JOIN build_tasks AS bt\n    ON tt.build_task_id = bt.id\n  INNER JOIN builds AS b\n    ON bt.build_id = b.id\n  INNER JOIN \n    (SELECT \n     tss.test_task_id, \n     MAX(tss.finish_ts) AS finished_at \n     FROM test_steps_stats AS tss\n     INNER JOIN test_tasks AS tt\n      ON tss.test_task_id = tt.id\n     INNER JOIN build_tasks AS bt\n      ON tt.build_task_id = bt.id\n     INNER JOIN builds AS b\n      ON bt.build_id = b.id\n      WHERE b.id = $build_id\n     GROUP BY tss.test_task_id) AS tf\n    ON tf.test_task_id = tt.id\n  WHERE b.id = $build_id\n  ) AS t;",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Tests duration",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 33,
+      "options": {
+        "displayMode": "basic",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": false
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  enum.value AS \"step name\",\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n  ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (enum.value); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Test duration (group by test step)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {}
+        }
+      ],
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": [],
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 0,
+        "y": 33
+      },
+      "id": 35,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "donut",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  aenum.value AS arch,\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN build_tasks as bt\n  ON tt.build_task_id = bt.id\nINNER JOIN arch_enum as aenum\n  ON bt.arch_id = aenum.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (aenum.value); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
+        }
+      ],
+      "title": "Tests duration (group by arch)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {
+            "mappings": []
+          }
+        }
+      ],
+      "type": "piechart"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": [],
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 16,
+        "x": 8,
+        "y": 33
+      },
+      "id": 37,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "donut",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  tt.package_fullname AS \"package name\",\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN build_tasks AS bt\n  ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (tt.package_fullname); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
+        }
+      ],
+      "title": "Tests duration (group by package)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {
+            "mappings": []
+          }
+        }
+      ],
+      "type": "piechart"
+    },
     {
       "collapsed": true,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 21
+        "y": 39
       },
       "id": 18,
       "panels": [
@@ -2079,7 +2548,7 @@
             "h": 9,
             "w": 24,
             "x": 0,
-            "y": 3
+            "y": 22
           },
           "id": 2,
           "options": {
@@ -2286,7 +2755,7 @@
             "h": 7,
             "w": 24,
             "x": 0,
-            "y": 12
+            "y": 31
           },
           "id": 25,
           "options": {
@@ -2367,7 +2836,374 @@
           "type": "table"
         }
       ],
-      "title": "Build tasks",
+      "title": "Details: build tasks",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 40
+      },
+      "id": 39,
+      "panels": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "left",
+                "displayMode": "auto",
+                "filterable": true,
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "duration"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "s"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "package"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 322
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "id"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 93
+                  },
+                  {
+                    "id": "links",
+                    "value": [
+                      {
+                        "title": "",
+                        "url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
+                      }
+                    ]
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "revision"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 62
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "build task id"
+                },
+                "properties": [
+                  {
+                    "id": "links",
+                    "value": [
+                      {
+                        "title": "",
+                        "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}"
+                      }
+                    ]
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 1
+          },
+          "id": 41,
+          "options": {
+            "footer": {
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "duration"
+              }
+            ]
+          },
+          "pluginVersion": "9.3.2",
+          "targets": [
+            {
+              "cacheDurationSeconds": 300,
+              "datasource": {
+                "type": "postgres",
+                "uid": "${DS_POSTGRESQL}"
+              },
+              "editorMode": "code",
+              "fields": [
+                {
+                  "jsonPath": ""
+                }
+              ],
+              "format": "table",
+              "method": "GET",
+              "queryParams": "",
+              "rawQuery": true,
+              "rawSql": "SELECT \n  DISTINCT tt.id,\n  tt.build_task_id as \"build task id\",\n  tt.package_fullname AS package,\n  tt.revision,\n  enum.value AS \"status\",\n  tt.started_at * 1000 AS \"started at\",\n  tf.finished_at * 1000 AS \"finished at\", \n  tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n  ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n  ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nINNER JOIN \n  (SELECT \n    tss.test_task_id, \n    MAX(tss.finish_ts) AS finished_at \n   FROM test_steps_stats AS tss\n   INNER JOIN test_tasks AS tt\n    ON tss.test_task_id = tt.id\n   INNER JOIN build_tasks AS bt\n    ON tt.build_task_id = bt.id\n   INNER JOIN builds AS b\n    ON bt.build_id = b.id\n   WHERE b.id = $build_id\n   GROUP BY tss.test_task_id) AS tf\n  ON tf.test_task_id = tt.id\nWHERE b.id = $build_id;",
+              "refId": "A",
+              "sql": {
+                "columns": [
+                  {
+                    "parameters": [],
+                    "type": "function"
+                  }
+                ],
+                "groupBy": [
+                  {
+                    "property": {
+                      "type": "string"
+                    },
+                    "type": "groupBy"
+                  }
+                ],
+                "limit": 50
+              },
+              "urlPath": ""
+            }
+          ],
+          "title": "Test tasks",
+          "transformations": [
+            {
+              "id": "convertFieldType",
+              "options": {
+                "conversions": [
+                  {
+                    "destinationType": "time",
+                    "targetField": "started at"
+                  },
+                  {
+                    "destinationType": "time",
+                    "targetField": "finished at"
+                  }
+                ],
+                "fields": {}
+              }
+            }
+          ],
+          "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "left",
+                "displayMode": "auto",
+                "filterable": true,
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "test task id"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 159
+                  },
+                  {
+                    "id": "links",
+                    "value": [
+                      {
+                        "title": "",
+                        "url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
+                      }
+                    ]
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "duration"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "s"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "package name"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 408
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 24,
+            "x": 0,
+            "y": 8
+          },
+          "id": 43,
+          "options": {
+            "footer": {
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "finished"
+              }
+            ]
+          },
+          "pluginVersion": "9.3.2",
+          "targets": [
+            {
+              "cacheDurationSeconds": 300,
+              "datasource": {
+                "type": "postgres",
+                "uid": "${DS_POSTGRESQL}"
+              },
+              "editorMode": "code",
+              "fields": [
+                {
+                  "jsonPath": ""
+                }
+              ],
+              "format": "table",
+              "method": "GET",
+              "queryParams": "",
+              "rawQuery": true,
+              "rawSql": "SELECT \n  tss.test_task_id AS \"test task id\", \n  tt.package_fullname AS \"package name\",\n  enum.value AS \"step name\",\n  tss.start_ts * 1000 AS started,\n  tss.finish_ts * 1000 AS finished,\n  tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n  ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n  ON bt.build_id = b.id\nWHERE b.id = $build_id\n",
+              "refId": "A",
+              "sql": {
+                "columns": [
+                  {
+                    "parameters": [],
+                    "type": "function"
+                  }
+                ],
+                "groupBy": [
+                  {
+                    "property": {
+                      "type": "string"
+                    },
+                    "type": "groupBy"
+                  }
+                ],
+                "limit": 50
+              },
+              "urlPath": ""
+            }
+          ],
+          "title": "Test steps",
+          "transformations": [
+            {
+              "id": "convertFieldType",
+              "options": {
+                "conversions": [
+                  {
+                    "destinationType": "time",
+                    "targetField": "started"
+                  },
+                  {
+                    "destinationType": "time",
+                    "targetField": "finished"
+                  }
+                ],
+                "fields": {}
+              }
+            }
+          ],
+          "type": "table"
+        }
+      ],
+      "title": "Details: tests",
       "type": "row"
     }
   ],
@@ -2406,6 +3242,6 @@
   "timezone": "",
   "title": "Build details",
   "uid": "dmVtrz-4k",
-  "version": 21,
+  "version": 34,
   "weekStart": ""
 }

build_analytics/grafana-dashbords/Build task details.json

@@ -35,6 +35,12 @@
       "name": "Grafana",
       "version": "9.3.2"
     },
+    {
+      "type": "panel",
+      "id": "piechart",
+      "name": "Pie chart",
+      "version": ""
+    },
     {
       "type": "datasource",
       "id": "postgres",
@@ -77,6 +83,17 @@
   "links": [],
   "liveNow": false,
   "panels": [
+    {
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 28,
+      "title": "Build task",
+      "type": "row"
+    },
     {
       "datasource": {
         "type": "postgres",
@@ -143,7 +160,7 @@
         "h": 3,
         "w": 24,
         "x": 0,
-        "y": 0
+        "y": 1
       },
       "id": 5,
       "options": {
@@ -196,7 +213,7 @@
           "urlPath": ""
         }
       ],
-      "title": "Task summary",
+      "title": "Build task",
       "transformations": [
         {
           "id": "convertFieldType",
@@ -259,10 +276,10 @@
         ]
       },
       "gridPos": {
-        "h": 9,
+        "h": 8,
         "w": 24,
         "x": 0,
-        "y": 3
+        "y": 4
       },
       "id": 12,
       "options": {
@@ -629,14 +646,219 @@
       "type": "bargauge"
     },
     {
-      "collapsed": true,
+      "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
         "y": 12
       },
-      "id": 8,
+      "id": 22,
+      "panels": [],
+      "title": "Test tasks",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 11,
+        "x": 0,
+        "y": 13
+      },
+      "id": 24,
+      "options": {
+        "displayMode": "basic",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": false
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  enum.value AS \"step name\",\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task\nGROUP BY (enum.value); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Test duration (group by test step)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {}
+        }
+      ],
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_POSTGRESQL}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            }
+          },
+          "mappings": [],
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 13,
+        "x": 11,
+        "y": 13
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "values": [
+            "value"
+          ]
+        },
+        "pieType": "donut",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "9.3.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "editorMode": "code",
+          "format": "table",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  tt.package_fullname AS \"package name\",\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nWHERE tt.build_task_id = $build_task\nGROUP BY (tt.package_fullname); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          }
+        }
+      ],
+      "title": "Tests duration (group by package)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {
+            "mappings": []
+          }
+        }
+      ],
+      "type": "piechart"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 20,
       "panels": [
         {
           "datasource": {
@@ -650,7 +872,7 @@
                 "mode": "thresholds"
               },
               "custom": {
-                "align": "auto",
+                "align": "left",
                 "displayMode": "auto",
                 "filterable": true,
                 "inspect": true
@@ -712,7 +934,7 @@
             "h": 5,
             "w": 24,
             "x": 0,
-            "y": 13
+            "y": 22
           },
           "id": 2,
           "options": {
@@ -750,7 +972,7 @@
               "method": "GET",
               "queryParams": "",
               "rawQuery": true,
-              "rawSql": "SELECT \n  enum.value,\n  stats.start_ts * 1000 AS started,\n  stats.end_ts * 1000 AS finished,\n  stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n  ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
+              "rawSql": "SELECT \n  enum.value as step,\n  stats.start_ts * 1000 AS started,\n  stats.end_ts * 1000 AS finished,\n  stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n  ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
               "refId": "web node stats",
               "sql": {
                 "columns": [
@@ -805,7 +1027,7 @@
                 "mode": "thresholds"
               },
               "custom": {
-                "align": "auto",
+                "align": "left",
                 "displayMode": "auto",
                 "filterable": true,
                 "inspect": true
@@ -852,10 +1074,10 @@
             ]
           },
           "gridPos": {
-            "h": 11,
+            "h": 8,
             "w": 24,
             "x": 0,
-            "y": 18
+            "y": 27
           },
           "id": 3,
           "options": {
@@ -893,7 +1115,7 @@
               "method": "GET",
               "queryParams": "",
               "rawQuery": true,
-              "rawSql": "SELECT enum.value,\n  stats.start_ts * 1000 AS started,\n  stats.end_ts * 1000 AS finished,\n  stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n  ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
+              "rawSql": "SELECT enum.value AS step,\n  stats.start_ts * 1000 AS started,\n  stats.end_ts * 1000 AS finished,\n  stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n  ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
               "refId": "web node stats",
               "sql": {
                 "columns": [
@@ -937,7 +1159,357 @@
           "type": "table"
         }
       ],
-      "title": "Details",
+      "title": "Details: build task",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 30,
+      "panels": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "left",
+                "displayMode": "auto",
+                "filterable": true,
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "package"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 389
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "id"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 72
+                  },
+                  {
+                    "id": "links",
+                    "value": [
+                      {
+                        "title": "",
+                        "url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
+                      }
+                    ]
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "revision"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 119
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "duration"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "s"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 5,
+            "w": 24,
+            "x": 0,
+            "y": 23
+          },
+          "id": 26,
+          "options": {
+            "footer": {
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "id"
+              }
+            ]
+          },
+          "pluginVersion": "9.3.2",
+          "targets": [
+            {
+              "cacheDurationSeconds": 300,
+              "datasource": {
+                "type": "postgres",
+                "uid": "${DS_POSTGRESQL}"
+              },
+              "editorMode": "code",
+              "fields": [
+                {
+                  "jsonPath": ""
+                }
+              ],
+              "format": "table",
+              "method": "GET",
+              "queryParams": "",
+              "rawQuery": true,
+              "rawSql": "SELECT \n  DISTINCT tt.id,\n  tt.package_fullname AS package,\n  tt.revision,\n  enum.value AS \"status\",\n  tt.started_at * 1000 AS \"started at\",\n  tf.finished_at * 1000 AS \"finished at\", \n  tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n  ON tt.status_id = enum.id\nINNER JOIN \n  (SELECT \n    tss.test_task_id, \n    MAX(tss.finish_ts) AS finished_at \n   FROM test_steps_stats AS tss\n   INNER JOIN test_tasks AS tt\n    ON tss.test_task_id = tt.id\n   WHERE tt.build_task_id = $build_task\n   GROUP BY tss.test_task_id) AS tf\n  ON tf.test_task_id = tt.id\nWHERE tt.build_task_id = $build_task; ",
+              "refId": "A",
+              "sql": {
+                "columns": [
+                  {
+                    "parameters": [],
+                    "type": "function"
+                  }
+                ],
+                "groupBy": [
+                  {
+                    "property": {
+                      "type": "string"
+                    },
+                    "type": "groupBy"
+                  }
+                ],
+                "limit": 50
+              },
+              "urlPath": ""
+            }
+          ],
+          "title": "Test tasks",
+          "transformations": [
+            {
+              "id": "convertFieldType",
+              "options": {
+                "conversions": [
+                  {
+                    "destinationType": "time",
+                    "targetField": "started at"
+                  },
+                  {
+                    "destinationType": "time",
+                    "targetField": "finished at"
+                  }
+                ],
+                "fields": {}
+              }
+            }
+          ],
+          "type": "table"
+        },
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_POSTGRESQL}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "thresholds"
+              },
+              "custom": {
+                "align": "left",
+                "displayMode": "auto",
+                "filterable": true,
+                "inspect": false
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "test task id"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 159
+                  },
+                  {
+                    "id": "links",
+                    "value": [
+                      {
+                        "title": "",
+                        "url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
+                      }
+                    ]
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "duration"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "s"
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "package name"
+                },
+                "properties": [
+                  {
+                    "id": "custom.width",
+                    "value": 408
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 6,
+            "w": 24,
+            "x": 0,
+            "y": 28
+          },
+          "id": 16,
+          "options": {
+            "footer": {
+              "fields": "",
+              "reducer": [
+                "sum"
+              ],
+              "show": false
+            },
+            "showHeader": true,
+            "sortBy": [
+              {
+                "desc": true,
+                "displayName": "finished"
+              }
+            ]
+          },
+          "pluginVersion": "9.3.2",
+          "targets": [
+            {
+              "cacheDurationSeconds": 300,
+              "datasource": {
+                "type": "postgres",
+                "uid": "${DS_POSTGRESQL}"
+              },
+              "editorMode": "code",
+              "fields": [
+                {
+                  "jsonPath": ""
+                }
+              ],
+              "format": "table",
+              "method": "GET",
+              "queryParams": "",
+              "rawQuery": true,
+              "rawSql": "SELECT \n  tt.package_fullname AS \"package name\",\n  tss.test_task_id AS \"test task id\", \n  enum.value AS \"step name\",\n  tss.start_ts * 1000 AS started,\n  tss.finish_ts * 1000 AS finished,\n  tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task; ",
+              "refId": "A",
+              "sql": {
+                "columns": [
+                  {
+                    "parameters": [],
+                    "type": "function"
+                  }
+                ],
+                "groupBy": [
+                  {
+                    "property": {
+                      "type": "string"
+                    },
+                    "type": "groupBy"
+                  }
+                ],
+                "limit": 50
+              },
+              "urlPath": ""
+            }
+          ],
+          "title": "Test steps",
+          "transformations": [
+            {
+              "id": "convertFieldType",
+              "options": {
+                "conversions": [
+                  {
+                    "destinationType": "time",
+                    "targetField": "started"
+                  },
+                  {
+                    "destinationType": "time",
+                    "targetField": "finished"
+                  }
+                ],
+                "fields": {}
+              }
+            }
+          ],
+          "type": "table"
+        }
+      ],
+      "title": "Details: test tasks",
       "type": "row"
     }
   ],
@@ -975,6 +1547,6 @@
   "timezone": "",
   "title": "Build task details",
   "uid": "vtQClqxVk",
-  "version": 37,
+  "version": 61,
   "weekStart": ""
 }

build_analytics/grafana-dashbords/Test task details.json

@@ -0,0 +1,516 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_ALBS_ANALYTICS",
+      "label": "albs_analytics",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "postgres",
+      "pluginName": "PostgreSQL"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "panel",
+      "id": "bargauge",
+      "name": "Bar gauge",
+      "version": ""
+    },
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "9.3.6"
+    },
+    {
+      "type": "datasource",
+      "id": "postgres",
+      "name": "PostgreSQL",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_ALBS_ANALYTICS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "left",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "duration"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "s"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "package"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 253
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "id"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": []
+      },
+      "pluginVersion": "9.3.6",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_ALBS_ANALYTICS}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  DISTINCT tt.id,\n  tt.package_fullname AS package,\n  tt.revision,\n  enum.value AS \"status\",\n  tt.started_at * 1000 AS \"started at\",\n  tf.finished_at * 1000 AS \"finished at\", \n  tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n  ON tt.status_id = enum.id\nINNER JOIN \n  (SELECT \n    tss.test_task_id, \n    MAX(tss.finish_ts) AS finished_at \n   FROM test_steps_stats AS tss\n   INNER JOIN test_tasks AS tt\n    ON tss.test_task_id = tt.id\n   WHERE tt.id = $id\n   GROUP BY tss.test_task_id) AS tf\n  ON tf.test_task_id = tt.id\nWHERE tt.id = $id;",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Task info",
+      "transformations": [
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "destinationType": "time",
+                "targetField": "started at"
+              },
+              {
+                "destinationType": "time",
+                "targetField": "finished at"
+              }
+            ],
+            "fields": {}
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_ALBS_ANALYTICS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 3
+      },
+      "id": 8,
+      "options": {
+        "displayMode": "basic",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": false
+      },
+      "pluginVersion": "9.3.6",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_ALBS_ANALYTICS}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  enum.value AS \"step name\",\n  SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nWHERE tt.id = $id\nGROUP BY (enum.value); ",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Test duration (group by test step)",
+      "transformations": [
+        {
+          "id": "rowsToFields",
+          "options": {}
+        }
+      ],
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "${DS_ALBS_ANALYTICS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "left",
+            "displayMode": "auto",
+            "filterable": true,
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "test task id"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 159
+              },
+              {
+                "id": "links",
+                "value": [
+                  {
+                    "title": "",
+                    "url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
+                  }
+                ]
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "duration"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "s"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "package name"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 408
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 11
+      },
+      "id": 6,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": false,
+            "displayName": "started"
+          }
+        ]
+      },
+      "pluginVersion": "9.3.6",
+      "targets": [
+        {
+          "cacheDurationSeconds": 300,
+          "datasource": {
+            "type": "postgres",
+            "uid": "${DS_ALBS_ANALYTICS}"
+          },
+          "editorMode": "code",
+          "fields": [
+            {
+              "jsonPath": ""
+            }
+          ],
+          "format": "table",
+          "method": "GET",
+          "queryParams": "",
+          "rawQuery": true,
+          "rawSql": "SELECT \n  tt.package_fullname AS \"package name\",\n  tss.test_task_id AS \"test task id\", \n  enum.value AS \"step name\",\n  tss.start_ts * 1000 AS started,\n  tss.finish_ts * 1000 AS finished,\n  tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n  ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n  ON tss.stat_name_id = enum.id\nWHERE tt.id = $id",
+          "refId": "A",
+          "sql": {
+            "columns": [
+              {
+                "parameters": [],
+                "type": "function"
+              }
+            ],
+            "groupBy": [
+              {
+                "property": {
+                  "type": "string"
+                },
+                "type": "groupBy"
+              }
+            ],
+            "limit": 50
+          },
+          "urlPath": ""
+        }
+      ],
+      "title": "Test steps",
+      "transformations": [
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "destinationType": "time",
+                "targetField": "started"
+              },
+              {
+                "destinationType": "time",
+                "targetField": "finished"
+              }
+            ],
+            "fields": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "datasource": {
+          "type": "postgres",
+          "uid": "${DS_ALBS_ANALYTICS}"
+        },
+        "definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Test task id",
+        "multi": false,
+        "name": "id",
+        "options": [],
+        "query": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Test task details",
+  "uid": "8nFXlkB4z",
+  "version": 3,
+  "weekStart": ""
+}

build_analytics/migrations/1.sql

@@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS builds_finished_at
 ON builds(finished_at);
 
 
--- build_taks_enum
+-- build_tasks_enum
 CREATE TABLE IF NOT EXISTS build_task_status_enum(
     id INTEGER PRIMARY KEY,
     value VARCHAR(15)
@@ -53,7 +53,7 @@ CREATE TABLE web_node_stats_enum (
 );
 
 INSERT INTO web_node_stats_enum (id, value)
-VALUEs
+VALUES
     (0, 'build_done'),
     (1, 'logs_processing'),
     (2, 'packages_processing');

build_analytics/migrations/3.sql

@@ -0,0 +1,80 @@
+BEGIN;
+
+-- test_tasks_status_enum
+CREATE TABLE  test_tasks_status_enum(
+    id INTEGER PRIMARY KEY,
+    value VARCHAR(15)
+);
+
+INSERT INTO test_tasks_status_enum (id, value)
+VALUES
+    (1, 'created'),
+    (2, 'started'),
+    (3, 'completed'),
+    (4, 'failed');
+
+
+-- test_tasks
+CREATE TABLE test_tasks (
+    id INTEGER PRIMARY KEY,
+    build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
+    revision INTEGER,
+    status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
+    package_fullname VARCHAR(100),
+    started_at DOUBLE PRECISION
+);
+
+CREATE INDEX test_tasks_build_task_id
+ON test_tasks(build_task_id);
+
+CREATE INDEX test_tasks_build_status_id
+ON test_tasks(status_id);
+
+CREATE INDEX test_tasks_package_fullname
+ON test_tasks(package_fullname);
+
+
+-- test_steps_enum
+CREATE TABLE test_steps_enum (
+    id INTEGER PRIMARY KEY,
+    value VARCHAR(50)
+);
+
+INSERT INTO test_steps_enum (id, value)
+VALUES
+    (0, 'install_package'),
+    (1, 'stop_environment'),
+    (2, 'initial_provision'),
+    (3, 'start_environment'),
+    (4, 'uninstall_package'),
+    (5, 'initialize_terraform'),
+    (6, 'package_integrity_tests'),
+    (7, 'stop_environment');
+
+
+
+-- test_steps_stats
+CREATE TABLE test_steps_stats(
+    test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
+    stat_name_id  INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
+    start_ts DOUBLE PRECISION,
+    finish_ts DOUBLE PRECISION
+);
+
+ALTER TABLE test_steps_stats
+ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
+
+CREATE INDEX test_steps_stats_start_ts
+ON test_steps_stats(start_ts);
+
+CREATE INDEX test_steps_stats_end_ts
+ON test_steps_stats(finish_ts);
+
+-- increasing size of name field
+ALTER TABLE build_tasks ALTER COLUMN name TYPE varchar(150);
+
+
+UPDATE schema_version
+SET version = 3;
+
+COMMIT;

build_analytics/releases.txt

@@ -1,10 +0,0 @@
-0.1.0 (2023-03-01)
-First version
-
-0.2.0 (2023-03-15)
-- New parameter start_from
-- Moved to double persition for timestamps
-- Added metrics for build steps
-
-0.2.1 (2023-03-15)
- - Added canceled Build task status

releases.txt

@@ -0,0 +1,40 @@
+0.1.0 (2023-03-01)
+First version
+
+0.2.0 (2023-03-15)
+- New parameter start_from
+- Moved to double persition for timestamps
+- Added metrics for build steps
+
+0.2.1 (2023-03-15)
+ - Added canceled Build task status
+
+0.3.0 (2023-03-22)
+ - Added test tasks stats
+ - New config parameter: oldest_to_update_days
+
+0.3.1 (2023-03-22)
+ - db: bugfix with migration to version 3
+ - added info about api_timeout config parameter
+ - bugfix with processing of test tasks with new revision
+
+0.3.2 (2023-03-23)
+ - Bugfix ALBS-1060
+
+0.3.3 (2023-04-24)
+build-analytics
+  Improvements
+    - [ALBS-1077] start deleting builds that were removed from ALBS
+  Bugfixes
+    - 'Key error' when db_port/db_host is not set
+    - update_builds() ignoring opldest_to_update attribute
+    - [ALBS-1099] Test task started_at attribute is NULL
+    - Max recursion error in 'Test task details.json'
+
+0.3.4 (2023-05-12)
+build_analytics
+  - Bigfix ALBS-1111
+
+0.3.5 (2023-06-01)
+build_analytics:
+  ALBS-1103 start using persistent HTTP connections