From 4149925e8e14dd7db10d0e1ca1c4fb7d99b26aaf Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 20 Apr 2023 08:45:34 +0200 Subject: [PATCH 1/2] Added new features to release.txt --- build_analytics/releases.txt => releases.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename build_analytics/releases.txt => releases.txt (70%) diff --git a/build_analytics/releases.txt b/releases.txt similarity index 70% rename from build_analytics/releases.txt rename to releases.txt index becc74f..57fdf9c 100644 --- a/build_analytics/releases.txt +++ b/releases.txt @@ -19,4 +19,8 @@ First version - bugfix with processing of test tasks with new revision 0.3.2 (2023-03-23) - - Bugfix ALBS-1060 \ No newline at end of file + - Bugfix ALBS-1060 + +0.3.3 (IN PROGRESS) + - [ALBS-1099] build-analytics: change source of Test task started_at timestamp + - [ALBS-1077] build_analytics: start deleting builds that were removed from ALBS \ No newline at end of file -- 2.43.5 From 7a1d84a0764c34b913a0d210c0670729a095fa02 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 21 Apr 2023 09:31:58 +0200 Subject: [PATCH 2/2] ALBS-1099 - Test task started_at attribute now is set as oldest start_ts attribute among all test steps Various - grafana-dashbords/Test task details.json: fixed 'max stack call' error (added LITMIT 1000 to id query) - fix typo in config_default.yml --- build_analytics/build_analytics/api_client.py | 36 ++++++++++++------ build_analytics/config_default.yml | 2 +- .../grafana-dashbords/Test task details.json | 38 +++++++++---------- releases.txt | 5 ++- 4 files changed, 47 insertions(+), 34 deletions(-) diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index a6187ff..a16c8d6 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -1,7 +1,7 @@ from datetime import datetime import logging from urllib.parse import urljoin -from typing import Dict, List, Any +from typing import Dict, List, Any, Optional import requests @@ -104,7 +104,8 @@ class APIclient(): if not task['performance_stats']: logging.warning( "no perfomance_stats for build_id: %s, build_task_id: %s", build_id, task['id']) - stats = {'build_node_stats': {}, 'build_done_stats': {}} + stats: dict[str, Any] = { + 'build_node_stats': {}, 'build_done_stats': {}} else: stats = task['performance_stats'][0]['statistics'] @@ -161,17 +162,10 @@ class APIclient(): return result def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]], - build_task_id: int, - started_at: str = None) -> List[TestTask]: + build_task_id: int) -> List[TestTask]: result: List[TestTask] = [] for task in raw_tasks: if task['alts_response']: - try: - started_raw = task['alts_response']['stats']['started_at'] - except KeyError: - started_at = None - else: - started_at = datetime.fromisoformat(started_raw+TZ_OFFSET) try: stats_raw = task['alts_response']['stats'] except KeyError: @@ -179,7 +173,6 @@ class APIclient(): else: steps_stats = self.__parse_test_steps_stats(stats_raw) else: - started_at = None steps_stats = None params = { 'id': task['id'], @@ -189,7 +182,8 @@ class APIclient(): 'package_fullname': '_'.join([task['package_name'], task['package_version'], task['package_release']]), - 'started_at': started_at, + 'started_at': self.__get_test_task_started_at( + steps_stats) if steps_stats else None, 'steps_stats': steps_stats } @@ -209,3 +203,21 @@ class APIclient(): p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET) teast_steps_params[field_name] = TestStepStat(**p) return TestStepsStats(**teast_steps_params) + + def __get_test_task_started_at(self, stats: TestStepsStats) -> Optional[datetime]: + """ + getting started_at attribute for test by using oldest start_ts timestamp + among all test tasks steps + """ + if not stats: + return None + + start_ts = None + for field_name in stats.__fields__.keys(): + stat: TestStepStat = getattr(stats, field_name) + if not stat: + continue + if not start_ts or start_ts > stat.start_ts: + start_ts = stat.start_ts + + return start_ts diff --git a/build_analytics/config_default.yml b/build_analytics/config_default.yml index 5e5e1c3..7b2dad7 100644 --- a/build_analytics/config_default.yml +++ b/build_analytics/config_default.yml @@ -60,7 +60,7 @@ scrape_interval: 3600 # build_id to start populating empty db with # required: false # default: 5808 (first build with correct metrics) -start_from: +start_from: 5808 # oldest_to_update # oldest (in days) unfinished object (build/task/step...) that we will try to update diff --git a/build_analytics/grafana-dashbords/Test task details.json b/build_analytics/grafana-dashbords/Test task details.json index 2568011..15dfa87 100644 --- a/build_analytics/grafana-dashbords/Test task details.json +++ b/build_analytics/grafana-dashbords/Test task details.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_POSTGRESQL", - "label": "PostgreSQL", + "name": "DS_ALBS_ANALYTICS", + "label": "albs_analytics", "description": "", "type": "datasource", "pluginId": "postgres", @@ -21,7 +21,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.3.2" + "version": "9.3.6" }, { "type": "datasource", @@ -58,7 +58,7 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -68,7 +68,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "fieldConfig": { "defaults": { @@ -152,13 +152,13 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -216,7 +216,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "fieldConfig": { "defaults": { @@ -263,13 +263,13 @@ }, "showUnfilled": false }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -315,7 +315,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "fieldConfig": { "defaults": { @@ -409,18 +409,18 @@ "showHeader": true, "sortBy": [ { - "desc": true, - "displayName": "finished" + "desc": false, + "displayName": "started" } ] }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -485,16 +485,16 @@ "current": {}, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, - "definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC", + "definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;", "hide": 0, "includeAll": false, "label": "Test task id", "multi": false, "name": "id", "options": [], - "query": "SELECT id\nFROM test_tasks\nORDER BY id DESC", + "query": "SELECT id\nFROM test_tasks\nORDER BY id DESC\nLIMIT 1000;", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -511,6 +511,6 @@ "timezone": "", "title": "Test task details", "uid": "8nFXlkB4z", - "version": 8, + "version": 3, "weekStart": "" } \ No newline at end of file diff --git a/releases.txt b/releases.txt index 57fdf9c..5e0fe7f 100644 --- a/releases.txt +++ b/releases.txt @@ -22,5 +22,6 @@ First version - Bugfix ALBS-1060 0.3.3 (IN PROGRESS) - - [ALBS-1099] build-analytics: change source of Test task started_at timestamp - - [ALBS-1077] build_analytics: start deleting builds that were removed from ALBS \ No newline at end of file +build-analytics: + - [ALBS-1099] change source of Test task started_at timestamp + - [ALBS-1077] start deleting builds that were removed from ALBS \ No newline at end of file -- 2.43.5