From 08ec138942dbd39ee232a754497683251e7c3d5b Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 15 Mar 2023 11:28:12 +0100 Subject: [PATCH] added api and db functions --- build_analytics/build_analytics/api_client.py | 57 ++++++++++++- build_analytics/build_analytics/const.py | 19 ++++- build_analytics/build_analytics/db.py | 24 ++++++ .../build_analytics/extractor/extractor.py | 18 +++-- .../build_analytics/models/test_step_stat.py | 10 +++ .../models/test_step_stat_db.py | 10 +++ .../models/test_steps_stats.py | 37 +++++++++ .../build_analytics/models/test_task.py | 31 ++++++++ .../build_analytics/models/test_task_db.py | 17 ++++ .../grafana-dashbords}/Build analytics.json | 0 .../grafana-dashbords}/Build details.json | 0 .../Build task details.json | 0 build_analytics/migrations/3.sql | 79 +++++++++---------- 13 files changed, 251 insertions(+), 51 deletions(-) create mode 100644 build_analytics/build_analytics/models/test_step_stat.py create mode 100644 build_analytics/build_analytics/models/test_step_stat_db.py create mode 100644 build_analytics/build_analytics/models/test_steps_stats.py create mode 100644 build_analytics/build_analytics/models/test_task.py create mode 100644 build_analytics/build_analytics/models/test_task_db.py rename {grafana-dashbords => build_analytics/grafana-dashbords}/Build analytics.json (100%) rename {grafana-dashbords => build_analytics/grafana-dashbords}/Build details.json (100%) rename {grafana-dashbords => build_analytics/grafana-dashbords}/Build task details.json (100%) diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index 56a293b..2b112e0 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -1,16 +1,19 @@ from datetime import datetime import logging from urllib.parse import urljoin -from typing import Dict, List +from typing import Dict, List, Any import requests + from .models.build import Build from .models.build_task import BuildTask from .models.build_node_stats import BuildNodeStats from .models.build_stat import BuildStat from .models.web_node_stats import WebNodeStats - +from .models.test_task import TestTask +from .models.test_steps_stats import TestStepsStats +from .models.test_step_stat import TestStepStat TZ_OFFSET = '+00:00' @@ -138,3 +141,53 @@ class APIclient(): 'build_tasks': build_tasks} return Build(**params) + + def get_test_tasks(self, build_task_id: int) -> List[TestTask]: + ep = f'/api/v1/tests/{build_task_id}/latest' + url = urljoin(self.api_root, ep) + headers = {'accept': 'application/json'} + + response = requests.get( + url, headers=headers, timeout=self.timeout) + response.raise_for_status() + return self.__parse_test_tasks() + + def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]], + build_task_id: int, + started_at: str = None) -> List[TestTask]: + result: List[TestTask] = [] + for task in raw_tasks: + if task['alts_response']: + started_raw = task['alts_response']['stats']['started_at'] + started_at = datetime.fromisoformat(started_raw+TZ_OFFSET) + stats_raw = task['alts_response']['stats'] + results_raw = task['results']['tests'] + step_stats = self.__parse_test_steps_stats( + stats_raw, results_raw) + else: + started_at = None + step_stats = None + params = { + 'id': task['id'], + 'build_task_id': build_task_id, + 'revision': task['revision'], + 'status': task['status'], + 'package_fullname': '_'.join([raw_tasks['package_name'], + raw_tasks['package_version'], + raw_tasks['package_release']]), + 'started_at': started_at, + 'step_stats': step_stats + } + result.append(TestTask(**params)) + return result + + def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats: + teast_steps_params = {} + for field_name in TestStepsStats.__fields__.keys(): + try: + p = stats_raw[field_name] + except KeyError: + continue + p['success'] = results_raw[field_name]['success'] + teast_steps_params[field_name] = TestStepStat(**p) + return TestStepsStats(**teast_steps_params) diff --git a/build_analytics/build_analytics/const.py b/build_analytics/build_analytics/const.py index d00a342..d77ff24 100644 --- a/build_analytics/build_analytics/const.py +++ b/build_analytics/build_analytics/const.py @@ -3,7 +3,7 @@ from enum import IntEnum # supported schema version -DB_SCHEMA_VER = 2 +DB_SCHEMA_VER = 3 # ENUMS @@ -41,3 +41,20 @@ class BuildNodeStatsEnum(IntEnum): build_node_task = 6 cas_notarize_artifacts = 7 cas_source_authenticate = 8 + + +class TestTaskStatusEnum(IntEnum): + created = 1 + started = 2 + completed = 3 + failed = 4 + + +class TestStepEnum(IntEnum): + install_package = 0 + stop_enviroment = 1 + initial_provision = 2 + start_enviroment = 3 + uninstall_package = 4 + initialize_terraform = 5 + package_integrity_tests = 6 diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index b266249..0b60d0d 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB from .models.build_node_stat_db import BuildNodeStatDB from .models.db_config import DbConfig from .models.web_node_stat_db import WebNodeStatDB +from .models.test_task_db import TestTaskDB class DB(): @@ -229,3 +230,26 @@ class DB(): cur.execute(sql, (build_task_id, stat_name_id)) val = int(cur.fetchone()[0]) return val == 1 + + def insert_test_task(self, task: TestTaskDB): + # inserting test task itself + sql = ''' + INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at) + VALUES (%s, %s, %s, %s, %s, %s); + ''' + + cur = self.__conn.cursor() + cur.execute(sql, (task.id, task.build_task_id, task.status_id, + task.package_fullname, task.started_at)) + # inserting test steps stats + for ss in task.steps_stats: + sql = ''' + INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success) + VALUES + (%s, %s, %s, %s, %s); + ''' + cur.execute(sql, (ss.test_task_id, ss.stat_name_id, + ss.start_ts, ss.end_ts, ss.success)) + + # commiting changes + self.__conn.commit() diff --git a/build_analytics/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py index 3a3a953..4702810 100644 --- a/build_analytics/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -1,13 +1,13 @@ # pylint: disable=relative-beyond-top-level import logging -from typing import List, Dict +from typing import Dict, List -from ..models.extractor_config import ExtractorConfig -from ..const import BuildTaskEnum -from ..models.build import BuildTask -from ..db import DB from ..api_client import APIclient +from ..const import BuildTaskEnum +from ..db import DB +from ..models.build import BuildTask +from ..models.extractor_config import ExtractorConfig class Extractor: @@ -36,7 +36,7 @@ class Extractor: break # inserting build build tasks and build tasks statistics - logging.info("inserting %s", build.id) + logging.info('inserting %s', build.id) try: self.db.insert_build(build.as_db_model()) except Exception as error: # pylint: disable=broad-except @@ -45,6 +45,8 @@ class Extractor: continue for build_task in build.build_tasks: + logging.info('build %s: inserting build task %s', + build.id, build_task.id) try: self.db.insert_buildtask(build_task.as_db_model(), build_task.web_node_stats.as_db_model( @@ -52,8 +54,8 @@ class Extractor: build_task.build_node_stats.as_db_model( build_task.id)) except Exception as error: # pylint: disable=broad-except - logging.error('failed to insert build task %d: %s', - build_task.id, error, exc_info=True) + logging.error('build %s: failed to insert build task %d: %s', + build.id, build_task.id, error, exc_info=True) build_count += 1 page_num += 1 return build_count diff --git a/build_analytics/build_analytics/models/test_step_stat.py b/build_analytics/build_analytics/models/test_step_stat.py new file mode 100644 index 0000000..2632646 --- /dev/null +++ b/build_analytics/build_analytics/models/test_step_stat.py @@ -0,0 +1,10 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +class TestStepStat(BaseModel): + start_ts: Optional[datetime] = None + end_ts: Optional[datetime] = None + success: bool diff --git a/build_analytics/build_analytics/models/test_step_stat_db.py b/build_analytics/build_analytics/models/test_step_stat_db.py new file mode 100644 index 0000000..ddae876 --- /dev/null +++ b/build_analytics/build_analytics/models/test_step_stat_db.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel # pylint: disable=no-name-in-module +from typing import Optional + + +class TestStepStatDB(BaseModel): + test_task_id: int + stat_name_id: int + start_ts: Optional[float] = None + end_ts: Optional[float] = None + success: bool diff --git a/build_analytics/build_analytics/models/test_steps_stats.py b/build_analytics/build_analytics/models/test_steps_stats.py new file mode 100644 index 0000000..5518982 --- /dev/null +++ b/build_analytics/build_analytics/models/test_steps_stats.py @@ -0,0 +1,37 @@ +from typing import List, Optional + +from pydantic import BaseModel # pylint: disable=no-name-in-module + +from ..const import TestStepEnum +from .test_step_stat import TestStepStat +from .test_step_stat_db import TestStepStatDB + + +class TestStepsStats(BaseModel): + install_package: Optional[TestStepStat] = None + stop_environment: Optional[TestStepStat] = None + initial_provision: Optional[TestStepStat] = None + start_environment: Optional[TestStepStat] = None + uninstall_package: Optional[TestStepStat] = None + initialize_terraform: Optional[TestStepStat] = None + package_integrity_tests: Optional[TestStepStat] = None + + def as_db(self, test_task_id: int) -> List[TestStepStatDB]: + result = [] + for field_name in self.__fields__.keys(): + stats: TestStepStat = getattr(self, field_name) + if not stats: + continue + start_ts = stats.start_ts.timestamp() \ + if stats.start_ts else None + end_ts = stats.end_ts.timestamp() \ + if stats.end_ts else None + stat_name_id = TestStepEnum[field_name].value + + test_step_stat_db = TestStepStatDB(test_task_id=test_task_id, + stat_name_id=stat_name_id, + start_ts=start_ts, + end_ts=end_ts, + success=stats.success) + result.append(test_step_stat_db) + return result diff --git a/build_analytics/build_analytics/models/test_task.py b/build_analytics/build_analytics/models/test_task.py new file mode 100644 index 0000000..1b87fea --- /dev/null +++ b/build_analytics/build_analytics/models/test_task.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel # pylint: disable=no-name-in-module + +from .test_task_db import TestTaskDB +from .test_steps_stats import TestStepsStats + + +class TestTask(BaseModel): + id: int + build_task_id: int + revision: int + status: int + package_fullname: str + started_at: Optional[datetime] = None + steps_stats: TestStepsStats + + def as_db_model(self) -> TestTaskDB: + started_at = self.started_at.timestamp() \ + if self.started_at else None + params = { + 'id': self.id, + 'build_task_id': self.build_task_id, + 'revision': self.revision, + 'status': self.status, + 'package_fullname': self.package_fullname, + 'started_at': started_at, + 'steps_stats': self.step_stats.as_db(self.id) + } + return TestTaskDB(**params) diff --git a/build_analytics/build_analytics/models/test_task_db.py b/build_analytics/build_analytics/models/test_task_db.py new file mode 100644 index 0000000..736d054 --- /dev/null +++ b/build_analytics/build_analytics/models/test_task_db.py @@ -0,0 +1,17 @@ +from typing import List, Optional +from pydantic import BaseModel # pylint: disable=no-name-in-module + +from .test_step_stat_db import TestStepStatDB + + +class TestTaskDB(BaseModel): + """ + Test task as it received from/sent to database + """ + id: int + build_task_id: int + revision: int + status_id: int + package_fullname: str + started_at: float + steps_stats: List[TestStepStatDB] = None diff --git a/grafana-dashbords/Build analytics.json b/build_analytics/grafana-dashbords/Build analytics.json similarity index 100% rename from grafana-dashbords/Build analytics.json rename to build_analytics/grafana-dashbords/Build analytics.json diff --git a/grafana-dashbords/Build details.json b/build_analytics/grafana-dashbords/Build details.json similarity index 100% rename from grafana-dashbords/Build details.json rename to build_analytics/grafana-dashbords/Build details.json diff --git a/grafana-dashbords/Build task details.json b/build_analytics/grafana-dashbords/Build task details.json similarity index 100% rename from grafana-dashbords/Build task details.json rename to build_analytics/grafana-dashbords/Build task details.json diff --git a/build_analytics/migrations/3.sql b/build_analytics/migrations/3.sql index a0e1fc5..cc2b688 100644 --- a/build_analytics/migrations/3.sql +++ b/build_analytics/migrations/3.sql @@ -1,75 +1,74 @@ BEGIN; --- test_task_status_enum -CREATE TABLE test_task_status_enum( +-- test_tasks_status_enum +CREATE TABLE test_tasks_status_enum( id INTEGER PRIMARY KEY, value VARCHAR(15) ); - - - -INSERT INTO test_task_status_enum (id, value) +INSERT INTO test_tasks_status_enum (id, value) VALUES - (0, 'created'), - (1, 'started'), - (2, 'completed'), - (3, 'failed'); + (1, 'created'), + (2, 'started'), + (3, 'completed'), + (4, 'failed'); --- test_task -CREATE TABLE test_task ( +-- test_tasks +CREATE TABLE test_tasks ( id INTEGER PRIMARY KEY, build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, revision INTEGER, - status_id INTEGER REFERENCES test_task_status_enum(id) ON DELETE SET NULL, - package_fullname VARCHAR(100) + status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL, + package_fullname VARCHAR(100), + started_at DOUBLE PRECISION ); -CREATE INDEX test_task_build_task_id -ON test_task(build_task_id); +CREATE INDEX test_tasks_build_task_id +ON test_tasks(build_task_id); -CREATE INDEX test_task_build_status_id -ON test_task(status_id); +CREATE INDEX test_tasks_build_status_id +ON test_tasks(status_id); -CREATE INDEX test_task_package_fullname -ON test_task(package_fullname); +CREATE INDEX test_tasks_package_fullname +ON test_tasks(package_fullname); --- test_step_enum -CREATE TABLE test_step_enum ( +-- test_steps_enum +CREATE TABLE test_steps_enum ( id INTEGER PRIMARY KEY, value VARCHAR(50) ); -INSERT INTO test_step_enum (id, value) +INSERT INTO test_steps_enum (id, value) VALUES - (1, 'install_package'), - (2, 'stop_environment'), - (3, 'initial_provision'), - (4, 'start_environment'), - (5, 'uninstall_package'), - (6, 'initialize_terraform'), - (7, 'package_integrity_tests'); + (0, 'install_package'), + (1, 'stop_environment'), + (2, 'initial_provision'), + (3, 'start_environment'), + (4, 'uninstall_package'), + (5, 'initialize_terraform'), + (6, 'package_integrity_tests'); --- test_step -CREATE TABLE test_step_stats( +-- test_steps +CREATE TABLE test_steps_stats( test_task_id INTEGER, - stat_name_id INTEGER REFERENCES test_step_enum(id) ON DELETE SET NULL, + stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL, start_ts DOUBLE PRECISION, - end_ts DOUBLE PRECISION + end_ts DOUBLE PRECISION, + success BOOLEAN ); -ALTER TABLE test_step_stats -ADD CONSTRAINT test_step_stats_unique UNIQUE (test_task_id, stat_name_id); +ALTER TABLE test_steps_stats +ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id); -CREATE INDEX test_step_stats_start_ts -ON test_step_stats(start_ts); +CREATE INDEX test_steps_stats_start_ts +ON test_steps_stats(start_ts); -CREATE INDEX test_step_stats_end_ts -ON test_step_stats(end_ts); +CREATE INDEX test_steps_stats_end_ts +ON test_steps_stats(end_ts); UPDATE schema_version