kzhukov
/
albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

added api and db functions

This commit is contained in:
Kirill Zhukov 2023-03-15 11:28:12 +01:00
parent a93165420b
commit 08ec138942
13 changed files with 251 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
build_analytics/build_analytics/api_client.py
View File

@ -1,16 +1,19 @@
from datetime import datetime
import logging
from urllib.parse import urljoin
from typing import Dict, List
from typing import Dict, List, Any
import requests
from .models.build import Build
from .models.build_task import BuildTask
from .models.build_node_stats import BuildNodeStats
from .models.build_stat import BuildStat
from .models.web_node_stats import WebNodeStats
from .models.test_task import TestTask
from .models.test_steps_stats import TestStepsStats
from .models.test_step_stat import TestStepStat
TZ_OFFSET = '+00:00'
@ -138,3 +141,53 @@ class APIclient():
'build_tasks': build_tasks}
return Build(**params)
def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
ep = f'/api/v1/tests/{build_task_id}/latest'
url = urljoin(self.api_root, ep)
headers = {'accept': 'application/json'}
response = requests.get(
url, headers=headers, timeout=self.timeout)
response.raise_for_status()
return self.__parse_test_tasks()
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
build_task_id: int,
started_at: str = None) -> List[TestTask]:
result: List[TestTask] = []
for task in raw_tasks:
if task['alts_response']:
started_raw = task['alts_response']['stats']['started_at']
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
stats_raw = task['alts_response']['stats']
results_raw = task['results']['tests']
step_stats = self.__parse_test_steps_stats(
stats_raw, results_raw)
else:
started_at = None
step_stats = None
params = {
'id': task['id'],
'build_task_id': build_task_id,
'revision': task['revision'],
'status': task['status'],
'package_fullname': '_'.join([raw_tasks['package_name'],
raw_tasks['package_version'],
raw_tasks['package_release']]),
'started_at': started_at,
'step_stats': step_stats
}
result.append(TestTask(**params))
return result
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats:
teast_steps_params = {}
for field_name in TestStepsStats.__fields__.keys():
try:
p = stats_raw[field_name]
except KeyError:
continue
p['success'] = results_raw[field_name]['success']
teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params)

19
build_analytics/build_analytics/const.py
View File

@ -3,7 +3,7 @@
from enum import IntEnum
# supported schema version
DB_SCHEMA_VER = 2
DB_SCHEMA_VER = 3
# ENUMS
@ -41,3 +41,20 @@ class BuildNodeStatsEnum(IntEnum):
build_node_task = 6
cas_notarize_artifacts = 7
cas_source_authenticate = 8
class TestTaskStatusEnum(IntEnum):
created = 1
started = 2
completed = 3
failed = 4
class TestStepEnum(IntEnum):
install_package = 0
stop_enviroment = 1
initial_provision = 2
start_enviroment = 3
uninstall_package = 4
initialize_terraform = 5
package_integrity_tests = 6

24
build_analytics/build_analytics/db.py
View File

@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
from .models.build_node_stat_db import BuildNodeStatDB
from .models.db_config import DbConfig
from .models.web_node_stat_db import WebNodeStatDB
from .models.test_task_db import TestTaskDB
class DB():
@ -229,3 +230,26 @@ class DB():
cur.execute(sql, (build_task_id, stat_name_id))
val = int(cur.fetchone()[0])
return val == 1
def insert_test_task(self, task: TestTaskDB):
# inserting test task itself
sql = '''
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
VALUES (%s, %s, %s, %s, %s, %s);
'''
cur = self.__conn.cursor()
cur.execute(sql, (task.id, task.build_task_id, task.status_id,
task.package_fullname, task.started_at))
# inserting test steps stats
for ss in task.steps_stats:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
VALUES
(%s, %s, %s, %s, %s);
'''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.end_ts, ss.success))
# commiting changes
self.__conn.commit()

18
build_analytics/build_analytics/extractor/extractor.py
View File

@ -1,13 +1,13 @@
# pylint: disable=relative-beyond-top-level
import logging
from typing import List, Dict
from typing import Dict, List
from ..models.extractor_config import ExtractorConfig
from ..const import BuildTaskEnum
from ..models.build import BuildTask
from ..db import DB
from ..api_client import APIclient
from ..const import BuildTaskEnum
from ..db import DB
from ..models.build import BuildTask
from ..models.extractor_config import ExtractorConfig
class Extractor:
@ -36,7 +36,7 @@ class Extractor:
break
# inserting build build tasks and build tasks statistics
logging.info("inserting %s", build.id)
logging.info('inserting %s', build.id)
try:
self.db.insert_build(build.as_db_model())
except Exception as error: # pylint: disable=broad-except
@ -45,6 +45,8 @@ class Extractor:
continue
for build_task in build.build_tasks:
logging.info('build %s: inserting build task %s',
build.id, build_task.id)
try:
self.db.insert_buildtask(build_task.as_db_model(),
build_task.web_node_stats.as_db_model(
@ -52,8 +54,8 @@ class Extractor:
build_task.build_node_stats.as_db_model(
build_task.id))
except Exception as error: # pylint: disable=broad-except
logging.error('failed to insert build task %d: %s',
build_task.id, error, exc_info=True)
logging.error('build %s: failed to insert build task %d: %s',
build.id, build_task.id, error, exc_info=True)
build_count += 1
page_num += 1
return build_count

10
build_analytics/build_analytics/models/test_step_stat.py Normal file
View File

@ -0,0 +1,10 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
class TestStepStat(BaseModel):
start_ts: Optional[datetime] = None
end_ts: Optional[datetime] = None
success: bool

10
build_analytics/build_analytics/models/test_step_stat_db.py Normal file
View File

@ -0,0 +1,10 @@
from pydantic import BaseModel # pylint: disable=no-name-in-module
from typing import Optional
class TestStepStatDB(BaseModel):
test_task_id: int
stat_name_id: int
start_ts: Optional[float] = None
end_ts: Optional[float] = None
success: bool

37
build_analytics/build_analytics/models/test_steps_stats.py Normal file
View File

@ -0,0 +1,37 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from ..const import TestStepEnum
from .test_step_stat import TestStepStat
from .test_step_stat_db import TestStepStatDB
class TestStepsStats(BaseModel):
install_package: Optional[TestStepStat] = None
stop_environment: Optional[TestStepStat] = None
initial_provision: Optional[TestStepStat] = None
start_environment: Optional[TestStepStat] = None
uninstall_package: Optional[TestStepStat] = None
initialize_terraform: Optional[TestStepStat] = None
package_integrity_tests: Optional[TestStepStat] = None
def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
result = []
for field_name in self.__fields__.keys():
stats: TestStepStat = getattr(self, field_name)
if not stats:
continue
start_ts = stats.start_ts.timestamp() \
if stats.start_ts else None
end_ts = stats.end_ts.timestamp() \
if stats.end_ts else None
stat_name_id = TestStepEnum[field_name].value
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
stat_name_id=stat_name_id,
start_ts=start_ts,
end_ts=end_ts,
success=stats.success)
result.append(test_step_stat_db)
return result

31
build_analytics/build_analytics/models/test_task.py Normal file
View File

@ -0,0 +1,31 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_task_db import TestTaskDB
from .test_steps_stats import TestStepsStats
class TestTask(BaseModel):
id: int
build_task_id: int
revision: int
status: int
package_fullname: str
started_at: Optional[datetime] = None
steps_stats: TestStepsStats
def as_db_model(self) -> TestTaskDB:
started_at = self.started_at.timestamp() \
if self.started_at else None
params = {
'id': self.id,
'build_task_id': self.build_task_id,
'revision': self.revision,
'status': self.status,
'package_fullname': self.package_fullname,
'started_at': started_at,
'steps_stats': self.step_stats.as_db(self.id)
}
return TestTaskDB(**params)

17
build_analytics/build_analytics/models/test_task_db.py Normal file
View File

@ -0,0 +1,17 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_step_stat_db import TestStepStatDB
class TestTaskDB(BaseModel):
"""
Test task as it received from/sent to database
"""
id: int
build_task_id: int
revision: int
status_id: int
package_fullname: str
started_at: float
steps_stats: List[TestStepStatDB] = None

0
grafana-dashbords/Build analytics.json → build_analytics/grafana-dashbords/Build analytics.json
View File

0
grafana-dashbords/Build details.json → build_analytics/grafana-dashbords/Build details.json
View File

0
grafana-dashbords/Build task details.json → build_analytics/grafana-dashbords/Build task details.json
View File

79
build_analytics/migrations/3.sql
View File

@ -1,75 +1,74 @@
BEGIN;
-- test_task_status_enum
CREATE TABLE test_task_status_enum(
-- test_tasks_status_enum
CREATE TABLE test_tasks_status_enum(
id INTEGER PRIMARY KEY,
value VARCHAR(15)
);
INSERT INTO test_task_status_enum (id, value)
INSERT INTO test_tasks_status_enum (id, value)
VALUES
(0, 'created'),
(1, 'started'),
(2, 'completed'),
(3, 'failed');
(1, 'created'),
(2, 'started'),
(3, 'completed'),
(4, 'failed');
-- test_task
CREATE TABLE test_task (
-- test_tasks
CREATE TABLE test_tasks (
id INTEGER PRIMARY KEY,
build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
revision INTEGER,
status_id INTEGER REFERENCES test_task_status_enum(id) ON DELETE SET NULL,
package_fullname VARCHAR(100)
status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
package_fullname VARCHAR(100),
started_at DOUBLE PRECISION
);
CREATE INDEX test_task_build_task_id
ON test_task(build_task_id);
CREATE INDEX test_tasks_build_task_id
ON test_tasks(build_task_id);
CREATE INDEX test_task_build_status_id
ON test_task(status_id);
CREATE INDEX test_tasks_build_status_id
ON test_tasks(status_id);
CREATE INDEX test_task_package_fullname
ON test_task(package_fullname);
CREATE INDEX test_tasks_package_fullname
ON test_tasks(package_fullname);
-- test_step_enum
CREATE TABLE test_step_enum (
-- test_steps_enum
CREATE TABLE test_steps_enum (
id INTEGER PRIMARY KEY,
value VARCHAR(50)
);
INSERT INTO test_step_enum (id, value)
INSERT INTO test_steps_enum (id, value)
VALUES
(1, 'install_package'),
(2, 'stop_environment'),
(3, 'initial_provision'),
(4, 'start_environment'),
(5, 'uninstall_package'),
(6, 'initialize_terraform'),
(7, 'package_integrity_tests');
(0, 'install_package'),
(1, 'stop_environment'),
(2, 'initial_provision'),
(3, 'start_environment'),
(4, 'uninstall_package'),
(5, 'initialize_terraform'),
(6, 'package_integrity_tests');
-- test_step
CREATE TABLE test_step_stats(
-- test_steps
CREATE TABLE test_steps_stats(
test_task_id INTEGER,
stat_name_id INTEGER REFERENCES test_step_enum(id) ON DELETE SET NULL,
stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL,
start_ts DOUBLE PRECISION,
end_ts DOUBLE PRECISION
end_ts DOUBLE PRECISION,
success BOOLEAN
);
ALTER TABLE test_step_stats
ADD CONSTRAINT test_step_stats_unique UNIQUE (test_task_id, stat_name_id);
ALTER TABLE test_steps_stats
ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
CREATE INDEX test_step_stats_start_ts
ON test_step_stats(start_ts);
CREATE INDEX test_steps_stats_start_ts
ON test_steps_stats(start_ts);
CREATE INDEX test_step_stats_end_ts
ON test_step_stats(end_ts);
CREATE INDEX test_steps_stats_end_ts
ON test_steps_stats(end_ts);
UPDATE schema_version