ALBS-1099 (#4)

Co-authored-by: Kirill Zhukov <kzhukov@cloudlinux.com>
Reviewed-on: #4

build_analytics/build_analytics/api_client.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 import logging
 from urllib.parse import urljoin
-from typing import Dict, List, Any
+from typing import Dict, List, Any, Optional
 
 import requests
 
@@ -104,7 +104,8 @@ class APIclient():
                 if not task['performance_stats']:
                     logging.warning(
                         "no perfomance_stats for build_id: %s, build_task_id: %s", build_id, task['id'])
-                    stats = {'build_node_stats': {}, 'build_done_stats': {}}
+                    stats: dict[str, Any] = {
+                        'build_node_stats': {}, 'build_done_stats': {}}
                 else:
                     stats = task['performance_stats'][0]['statistics']
 
@@ -161,17 +162,10 @@ class APIclient():
         return result
 
     def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
-                           build_task_id: int,
-                           started_at: str = None) -> List[TestTask]:
+                           build_task_id: int) -> List[TestTask]:
         result: List[TestTask] = []
         for task in raw_tasks:
             if task['alts_response']:
-                try:
-                    started_raw = task['alts_response']['stats']['started_at']
-                except KeyError:
-                    started_at = None
-                else:
-                    started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
                 try:
                     stats_raw = task['alts_response']['stats']
                 except KeyError:
@@ -179,7 +173,6 @@ class APIclient():
                 else:
                     steps_stats = self.__parse_test_steps_stats(stats_raw)
             else:
-                started_at = None
                 steps_stats = None
             params = {
                 'id': task['id'],
@@ -189,7 +182,8 @@ class APIclient():
                 'package_fullname': '_'.join([task['package_name'],
                                               task['package_version'],
                                               task['package_release']]),
-                'started_at': started_at,
+                'started_at': self.__get_test_task_started_at(
+                    steps_stats) if steps_stats else None,
                 'steps_stats': steps_stats
             }
 
@@ -209,3 +203,21 @@ class APIclient():
                     p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
             teast_steps_params[field_name] = TestStepStat(**p)
         return TestStepsStats(**teast_steps_params)
+
+    def __get_test_task_started_at(self, stats: TestStepsStats) -> Optional[datetime]:
+        """
+        getting started_at attribute for test by using oldest start_ts timestamp
+        among all test tasks steps
+        """
+        if not stats:
+            return None
+
+        start_ts = None
+        for field_name in stats.__fields__.keys():
+            stat: TestStepStat = getattr(stats, field_name)
+            if not stat:
+                continue
+            if not start_ts or start_ts > stat.start_ts:
+                start_ts = stat.start_ts
+
+        return start_ts

build_analytics/config_default.yml

@@ -60,7 +60,7 @@ scrape_interval: 3600
 # build_id to start populating empty db with
 # required: false
 # default: 5808 (first build with correct metrics)
-start_from:
+start_from: 5808
 
 # oldest_to_update
 # oldest (in days) unfinished object (build/task/step...) that we will try to update

build_analytics/grafana-dashbords/Test task details.json

@@ -1,8 +1,8 @@
 {
   "__inputs": [
     {
-      "name": "DS_POSTGRESQL",
-      "label": "PostgreSQL",
+      "name": "DS_ALBS_ANALYTICS",
+      "label": "albs_analytics",
       "description": "",
       "type": "datasource",
       "pluginId": "postgres",
@@ -21,7 +21,7 @@
       "type": "grafana",
       "id": "grafana",
       "name": "Grafana",
-      "version": "9.3.2"
+      "version": "9.3.6"
     },
     {
       "type": "datasource",
@@ -58,7 +58,7 @@
       }
     ]
   },
-  "editable": false,
+  "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": null,
@@ -68,7 +68,7 @@
     {
       "datasource": {
         "type": "postgres",
-        "uid": "${DS_POSTGRESQL}"
+        "uid": "${DS_ALBS_ANALYTICS}"
       },
       "fieldConfig": {
         "defaults": {
@@ -152,13 +152,13 @@
         "showHeader": true,
         "sortBy": []
       },
-      "pluginVersion": "9.3.2",
+      "pluginVersion": "9.3.6",
       "targets": [
         {
           "cacheDurationSeconds": 300,
           "datasource": {
             "type": "postgres",
-            "uid": "${DS_POSTGRESQL}"
+            "uid": "${DS_ALBS_ANALYTICS}"
           },
           "editorMode": "code",
           "fields": [
@@ -216,7 +216,7 @@
     {
       "datasource": {
         "type": "postgres",
-        "uid": "${DS_POSTGRESQL}"
+        "uid": "${DS_ALBS_ANALYTICS}"
       },
       "fieldConfig": {
         "defaults": {
@@ -263,13 +263,13 @@
         },
         "showUnfilled": false
       },
-      "pluginVersion": "9.3.2",
+      "pluginVersion": "9.3.6",
       "targets": [
         {
           "cacheDurationSeconds": 300,
           "datasource": {
             "type": "postgres",
-            "uid": "${DS_POSTGRESQL}"
+            "uid": "${DS_ALBS_ANALYTICS}"
           },
           "editorMode": "code",
           "fields": [
@@ -315,7 +315,7 @@
     {
       "datasource": {
         "type": "postgres",
-        "uid": "${DS_POSTGRESQL}"
+        "uid": "${DS_ALBS_ANALYTICS}"
       },
       "fieldConfig": {
         "defaults": {
@@ -409,18 +409,18 @@
         "showHeader": true,
         "sortBy": [
           {
-            "desc": true,
-            "displayName": "finished"
+            "desc": false,
+            "displayName": "started"
           }
         ]
       },
-      "pluginVersion": "9.3.2",
+      "pluginVersion": "9.3.6",
       "targets": [
         {
           "cacheDurationSeconds": 300,
           "datasource": {
             "type": "postgres",
-            "uid": "${DS_POSTGRESQL}"
+            "uid": "${DS_ALBS_ANALYTICS}"
           },
           "editorMode": "code",
           "fields": [
@@ -485,16 +485,16 @@
         "current": {},
         "datasource": {
           "type": "postgres",
-          "uid": "${DS_POSTGRESQL}"
+          "uid": "${DS_ALBS_ANALYTICS}"
         },
-        "definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
+        "definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;",
         "hide": 0,
         "includeAll": false,
         "label": "Test task id",
         "multi": false,
         "name": "id",
         "options": [],
-        "query": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
+        "query": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -511,6 +511,6 @@
   "timezone": "",
   "title": "Test task details",
   "uid": "8nFXlkB4z",
-  "version": 8,
+  "version": 3,
   "weekStart": ""
 }

releases.txt

@@ -19,4 +19,9 @@ First version
  - bugfix with processing of test tasks with new revision
 
 0.3.2 (2023-03-23)
- - Bugfix ALBS-1060
+ - Bugfix ALBS-1060
+
+0.3.3 (IN PROGRESS)
+build-analytics:
+ - [ALBS-1099] change source of Test task started_at timestamp
+ - [ALBS-1077] start deleting builds that were removed from ALBS