kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

Merge pull request 'ALBS-1043' (#2) from ALBS-1043 into main

Browse Source 
Reviewed-on: #2
This commit is contained in:
kzhukov 2023-03-22 10:35:03 +00:00
commit bd74c99a7d
20 changed files with 5184 additions and 1948 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
build_analytics/build_analytics/api_client.py
View File

@ -1,16 +1,19 @@
from datetime import datetime
import logging
from urllib.parse import urljoin
from typing import Dict, List
from typing import Dict, List, Any
import requests
from .models.build import Build
from .models.build_task import BuildTask
from .models.build_node_stats import BuildNodeStats
from .models.build_stat import BuildStat
from .models.web_node_stats import WebNodeStats
from .models.test_task import TestTask
from .models.test_steps_stats import TestStepsStats
from .models.test_step_stat import TestStepStat
TZ_OFFSET = '+00:00'
@ -138,3 +141,63 @@ class APIclient():
'build_tasks': build_tasks}
return Build(**params)
def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
ep = f'/api/v1/tests/{build_task_id}/latest'
url = urljoin(self.api_root, ep)
headers = {'accept': 'application/json'}
response = requests.get(
url, headers=headers, timeout=self.timeout)
response.raise_for_status()
return self.__parse_test_tasks(response.json(), build_task_id)
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
build_task_id: int,
started_at: str = None) -> List[TestTask]:
result: List[TestTask] = []
for task in raw_tasks:
if task['alts_response']:
try:
started_raw = task['alts_response']['stats']['started_at']
except KeyError:
started_at = None
else:
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
try:
stats_raw = task['alts_response']['stats']
except KeyError:
steps_stats = None
else:
steps_stats = self.__parse_test_steps_stats(stats_raw)
else:
started_at = None
steps_stats = None
params = {
'id': task['id'],
'build_task_id': build_task_id,
'revision': task['revision'],
'status': task['status'],
'package_fullname': '_'.join([task['package_name'],
task['package_version'],
task['package_release']]),
'started_at': started_at,
'steps_stats': steps_stats
}
result.append(TestTask(**params))
return result
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
teast_steps_params = {}
for field_name in TestStepsStats.__fields__.keys():
try:
p = stats_raw[field_name]
except KeyError:
continue
# there are must be a better way
for k in ['start_ts', 'finish_ts']:
if k in p:
p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params)

20
build_analytics/build_analytics/const.py
View File

@ -3,7 +3,7 @@
from enum import IntEnum
# supported schema version
DB_SCHEMA_VER = 2
DB_SCHEMA_VER = 3
# ENUMS
@ -41,3 +41,21 @@ class BuildNodeStatsEnum(IntEnum):
build_node_task = 6
cas_notarize_artifacts = 7
cas_source_authenticate = 8
class TestTaskStatusEnum(IntEnum):
created = 1
started = 2
completed = 3
failed = 4
class TestStepEnum(IntEnum):
install_package = 0
stop_enviroment = 1
initial_provision = 2
start_environment = 3
uninstall_package = 4
initialize_terraform = 5
package_integrity_tests = 6
stop_environment = 7

123
build_analytics/build_analytics/db.py
View File

@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
from .models.build_node_stat_db import BuildNodeStatDB
from .models.db_config import DbConfig
from .models.web_node_stat_db import WebNodeStatDB
from .models.test_task_db import TestTaskDB
class DB():
@ -25,6 +26,17 @@ class DB():
def __del__(self):
self.close_conn()
def build_exists(self, build_id: int) -> bool:
sql = f'''
SELECT COUNT(id)
FROM builds
WHERE id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (build_id,))
val = int(cur.fetchone()[0])
return val == 1
def insert_build(self, build: BuildDB):
sql = '''
INSERT INTO builds(id, url, created_at, finished_at)
@ -99,7 +111,7 @@ class DB():
self.__conn.commit()
return cur.rowcount
def get_unfinished_builds(self) -> Dict[int, Dict[int, int]]:
def get_unfinished_builds(self, not_before: datetime) -> Dict[int, Dict[int, int]]:
"""
Getting list of unfinished builds and build_tasks
Dict[build_id, Dict[build_task_id, task_status_id]]
@ -107,9 +119,10 @@ class DB():
res: Dict[int, Dict[int, int]] = {}
# getting unfinished builds
sql = 'SELECT id FROM builds where finished_at is NULL;'
sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
cur = self.__conn.cursor()
cur.execute(sql)
cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
for row in cur.fetchall():
res[row[0]] = {}
@ -159,9 +172,10 @@ class DB():
for stat in web_node_stats:
logging.debug(
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id,
if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='web_node_stats'):
table_name='web_node_stats',
column_name='build_task_id'):
sql = '''
UPDATE web_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -183,9 +197,10 @@ class DB():
for stat in build_node_stats:
logging.debug(
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id,
if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id,
table_name='build_node_stats'):
table_name='build_node_stats',
column_name='build_task_id'):
sql = '''
UPDATE build_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -219,13 +234,99 @@ class DB():
return None
return int(val[0])
def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool:
def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool:
sql = f'''
SELECT COUNT(build_task_id)
SELECT COUNT({column_name})
FROM {table_name}
WHERE build_task_id = %s AND stat_name_id = %s;
WHERE {column_name} = %s AND stat_name_id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (build_task_id, stat_name_id))
cur.execute(sql, (task_id, stat_name_id))
val = int(cur.fetchone()[0])
return val == 1
def insert_test_task(self, task: TestTaskDB):
cur = self.__conn.cursor()
# inserting test task itself
sql = '''
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
VALUES
(%s, %s, %s, %s, %s, %s);
'''
cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
task.package_fullname, task.started_at))
if task.steps_stats:
# inserting test steps stats
for ss in task.steps_stats:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
VALUES
(%s, %s, %s, %s);
'''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.finish_ts))
# commiting changes
self.__conn.commit()
def get_build_tasks_for_unfinished_tests(self, not_before: datetime) -> List[int]:
'''
getting build tasks id of unfinished test tasks
'''
cur = self.__conn.cursor()
sql = '''
SELECT DISTINCT bt.id
FROM build_tasks as bt
INNER JOIN test_tasks AS tt
ON bt.id = tt.build_task_id
WHERE tt.status_id < 3 AND bt.started_at > %s;
'''
cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()]
return result
def update_test_tasks(self, test_tasks: List[TestTaskDB]):
cur = self.__conn.cursor()
# test tasks
for task in test_tasks:
sql = '''
UPDATE test_tasks
SET revision = %s,
status_id = %s,
started_at = %s
WHERE id = %s;
'''
cur.execute(sql, (task.revision, task.status_id,
task.started_at, task.id))
assert cur.rowcount == 1
# test step
if not task.steps_stats:
continue
for s in task.steps_stats:
logging.info('test_task_id %s, stat_name_id %s',
s.test_task_id, s.stat_name_id)
if self.stat_exists(s.test_task_id,
s.stat_name_id,
'test_steps_stats',
'test_task_id'):
sql = '''
UPDATE test_steps_stats
SET start_ts = %s,
finish_ts = %s
WHERE test_task_id = %s AND stat_name_id = %s;
'''
cur.execute(sql, (s.start_ts, s.finish_ts,
s.test_task_id, s.stat_name_id))
assert cur.rowcount == 1
else:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
VALUES (%s, %s, %s, %s);
'''
cur.execute(sql, (s.test_task_id, s.stat_name_id,
s.start_ts, s.finish_ts))
# commiting changes
self.__conn.commit()

61
build_analytics/build_analytics/extractor/extractor.py
View File

@ -1,19 +1,18 @@
# pylint: disable=relative-beyond-top-level
import logging
from typing import List, Dict
from typing import Dict, List
from ..models.extractor_config import ExtractorConfig
from ..const import BuildTaskEnum
from ..models.build import BuildTask
from ..db import DB
from ..api_client import APIclient
from ..const import BuildTaskEnum
from ..db import DB
from ..models.build import BuildTask
from ..models.extractor_config import ExtractorConfig
class Extractor:
def __init__(self, config: ExtractorConfig, api: APIclient, db: DB):
self.start_from = config.start_from
self.oldest_build_age = config.oldest_build_age
self.config = config
self.api = api
self.db = db
@ -22,7 +21,7 @@ class Extractor:
page_num = 1
last_build_id = self.db.get_latest_build_id()
if not last_build_id:
last_build_id = self.start_from - 1
last_build_id = self.config.start_from - 1
logging.info("last_build_id: %s", last_build_id)
stop = False
@ -31,12 +30,16 @@ class Extractor:
for build in self.api.get_builds(page_num):
# check if we shoud stop processing build
if build.id <= last_build_id or \
build.created_at <= self.oldest_build_age:
build.created_at <= self.config.oldest_build_age:
stop = True
break
# some builds could move from one page to another
if self.db.build_exists(build_id=build.id):
continue
# inserting build build tasks and build tasks statistics
logging.info("inserting %s", build.id)
logging.info('inserting %s', build.id)
try:
self.db.insert_build(build.as_db_model())
except Exception as error: # pylint: disable=broad-except
@ -45,6 +48,8 @@ class Extractor:
continue
for build_task in build.build_tasks:
logging.info('build %s: inserting build task %s',
build.id, build_task.id)
try:
self.db.insert_buildtask(build_task.as_db_model(),
build_task.web_node_stats.as_db_model(
@ -52,16 +57,25 @@ class Extractor:
build_task.build_node_stats.as_db_model(
build_task.id))
except Exception as error: # pylint: disable=broad-except
logging.error('failed to insert build task %d: %s',
build_task.id, error, exc_info=True)
logging.error('build %s: failed to insert build task %d: %s',
build.id, build_task.id, error, exc_info=True)
logging.info(
'getting test tasks for build task %s', build_task.id)
test_tasks = self.api.get_test_tasks(build_task.id)
logging.info('received %d tests tasks', len(test_tasks))
for t in test_tasks:
logging.info(
'build task %s: inserting test task %s', build_task.id, t.id)
self.db.insert_test_task(t.as_db_model())
build_count += 1
page_num += 1
return build_count
def build_cleanup(self):
logging.info('Removing all buidls older then %s',
self.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
removed_count = self.db.cleanup_builds(self.oldest_build_age)
self.config.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
removed_count = self.db.cleanup_builds(self.config.oldest_build_age)
logging.info('removed %d entries', removed_count)
def __update_build_tasks(self, build_tasks: List[BuildTask],
@ -92,7 +106,8 @@ class Extractor:
def update_builds(self):
logging.info('Getting list of tasks from DB')
unfinished_tasks = self.db.get_unfinished_builds()
unfinished_tasks = self.db.get_unfinished_builds(
self.config.oldest_to_update)
for build_id, build_tasks_db in unfinished_tasks.items():
try:
logging.info('Getting status of build %d', build_id)
@ -114,3 +129,19 @@ class Extractor:
except Exception as err: # pylint: disable=broad-except
logging.error("Cant process build %d: %s",
build_id, err, exc_info=True)
def updating_test_tasks(self):
logging.info('getting build task ids of unfinished tests')
build_task_ids = self.db.get_build_tasks_for_unfinished_tests(
self.config.oldest_to_update)
for build_task_id in build_task_ids:
try:
logging.info('getting tests for build task %s', build_task_id)
tasks_api = self.api.get_test_tasks(build_task_id)
logging.info('updating test tasks')
tasks_db = [t.as_db_model() for t in tasks_api]
self.db.update_test_tasks(tasks_db)
except Exception as err: # pylint: disable=broad-except
logging.error(
'failed to update tests for %d build task: %s',
build_task_id, err, exc_info=True)

48
build_analytics/build_analytics/extractor/start.py
View File

@ -15,20 +15,6 @@ from ..models.extractor_config import ExtractorConfig
from ..models.db_config import DbConfig
def __get_oldest_build_age(config: dict) -> datetime:
oldest_build_age = datetime.now().astimezone() \
- timedelta(days=config['data_store_days'])
return oldest_build_age
def __get_db_config(config: dict) -> DbConfig:
return DbConfig(name=config['db_name'],
port=int(config['db_port']),
host=config['db_host'],
username=config['db_username'],
password=config['db_password'])
def __get_config(yml_path: str) -> ExtractorConfig:
"""
get_config loads yml file and generates instance
@ -37,8 +23,18 @@ def __get_config(yml_path: str) -> ExtractorConfig:
raw = yaml.safe_load(flr)
# adding new attrs
raw['oldest_build_age'] = __get_oldest_build_age(raw)
raw['db_config'] = __get_db_config(raw)
raw['oldest_build_age'] = datetime.now().astimezone() \
- timedelta(days=raw['data_store_days'])
raw['db_config'] = DbConfig(name=raw['db_name'],
port=int(raw['db_port']),
host=raw['db_host'],
username=raw['db_username'],
password=raw['db_password'])
if 'oldest_to_update_days' in raw:
raw['oldest_to_update_days'] = datetime.now().astimezone() \
- timedelta(days=raw['oldest_to_update_days'])
return ExtractorConfig(**raw)
@ -83,21 +79,29 @@ def start(yml_path: str):
logging.info(
'Build extaction was finished. %d builds were inserted', inserted_count)
logging.info('Starting old builds removal')
logging.info('starting old builds removal')
try:
extractor.build_cleanup()
except Exception as err: # pylint: disable=broad-except
logging.critical("Unhandled exception %s", err, exc_info=True)
logging.critical("unhandled exception %s", err, exc_info=True)
else:
logging.info('Cleanup finished')
logging.info('cleanup finished')
logging.info('Updating statuses of unfinished build tasks')
logging.info('updating statuses of unfinished build tasks')
try:
extractor.update_builds()
except Exception as err: # pylint: disable=broad-except
logging.critical("Unhandled exception %s", err, exc_info=True)
logging.critical("unhandled exception %s", err, exc_info=True)
else:
logging.info('Update finished')
logging.info('update finished')
logging.info('updating statuses of unfinished test tasks')
try:
extractor.updating_test_tasks()
except Exception as err: # pylint: disable=broad-except
logging.critical("unhandled exception %s", err, exc_info=True)
else:
logging.info('test tasks were updated')
extractor.db.close_conn()
logging.info("Extraction was finished")

6
build_analytics/build_analytics/models/extractor_config.py
View File

@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta
from pathlib import Path
from pydantic import HttpUrl, Field, BaseModel # pylint: disable=no-name-in-module
@ -11,6 +11,7 @@ LOG_FILE_DEFAULT = '/tmp/extractor.log'
API_DEFAULT = 30
SCRAPE_INTERVAL_DEFAULT = 3600
START_FROM_DEFAULT = 5808
OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=7)
class ExtractorConfig(BaseModel):
@ -32,3 +33,6 @@ class ExtractorConfig(BaseModel):
default=SCRAPE_INTERVAL_DEFAULT)
start_from: int = Field(description='build id to start populating empty db with',
default=START_FROM_DEFAULT)
oldest_to_update: datetime = \
Field(description='oldest unfinished object (build/task/step...) that we will try to update',
default=OLDEST_TO_UPDATE_DEFAULT)

2
build_analytics/build_analytics/models/sign_task_db.py
View File

@ -1,4 +1,4 @@
from pydantic import BaseModel
from pydantic import BaseModel # pylint: disable=no-name-in-module
class SignTaskDB(BaseModel):

9
build_analytics/build_analytics/models/test_step_stat.py Normal file
View File

@ -0,0 +1,9 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
class TestStepStat(BaseModel):
start_ts: Optional[datetime] = None
finish_ts: Optional[datetime] = None

9
build_analytics/build_analytics/models/test_step_stat_db.py Normal file
View File

@ -0,0 +1,9 @@
from pydantic import BaseModel # pylint: disable=no-name-in-module
from typing import Optional
class TestStepStatDB(BaseModel):
test_task_id: int
stat_name_id: int
start_ts: Optional[float] = None
finish_ts: Optional[float] = None

36
build_analytics/build_analytics/models/test_steps_stats.py Normal file
View File

@ -0,0 +1,36 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from ..const import TestStepEnum
from .test_step_stat import TestStepStat
from .test_step_stat_db import TestStepStatDB
class TestStepsStats(BaseModel):
install_package: Optional[TestStepStat] = None
stop_environment: Optional[TestStepStat] = None
initial_provision: Optional[TestStepStat] = None
start_environment: Optional[TestStepStat] = None
uninstall_package: Optional[TestStepStat] = None
initialize_terraform: Optional[TestStepStat] = None
package_integrity_tests: Optional[TestStepStat] = None
def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
result = []
for field_name in self.__fields__.keys():
stats: TestStepStat = getattr(self, field_name)
if not stats:
continue
start_ts = stats.start_ts.timestamp() \
if stats.start_ts else None
finish_ts = stats.finish_ts.timestamp() \
if stats.finish_ts else None
stat_name_id = TestStepEnum[field_name].value
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
stat_name_id=stat_name_id,
start_ts=start_ts,
finish_ts=finish_ts)
result.append(test_step_stat_db)
return result

31
build_analytics/build_analytics/models/test_task.py Normal file
View File

@ -0,0 +1,31 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_task_db import TestTaskDB
from .test_steps_stats import TestStepsStats
class TestTask(BaseModel):
id: int
build_task_id: int
revision: int
status: int
package_fullname: str
started_at: Optional[datetime] = None
steps_stats: Optional[TestStepsStats] = None
def as_db_model(self) -> TestTaskDB:
started_at = self.started_at.timestamp() \
if self.started_at else None
params = {
'id': self.id,
'build_task_id': self.build_task_id,
'revision': self.revision,
'status_id': self.status,
'package_fullname': self.package_fullname,
'started_at': started_at,
'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
}
return TestTaskDB(**params)

17
build_analytics/build_analytics/models/test_task_db.py Normal file
View File

@ -0,0 +1,17 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_step_stat_db import TestStepStatDB
class TestTaskDB(BaseModel):
"""
Test task as it received from/sent to database
"""
id: int
build_task_id: int
revision: int
status_id: int
package_fullname: str
started_at: Optional[float] = None
steps_stats: Optional[List[TestStepStatDB]] = None

8
build_analytics/config_default.yml
View File

@ -61,3 +61,11 @@ scrape_interval: 3600
# required: false
# default: 5808 (first build with correct metrics)
start_from:
# oldest_to_update
# oldest (in days) unfinished object (build/task/step...) that we will try to update
# required: false
# default: 7
oldest_to_update_days: 7

4606
grafana-dashbords/Build analytics.json → build_analytics/grafana-dashbords/Build analytics.json
View File

File diff suppressed because it is too large Load Diff

884
grafana-dashbords/Build details.json → build_analytics/grafana-dashbords/Build details.json
View File

@ -87,7 +87,7 @@
},
"id": 16,
"panels": [],
"title": "Summary",
"title": "Builds",
"type": "row"
},
{
@ -1261,19 +1261,6 @@
],
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 12
},
"id": 23,
"panels": [],
"title": "Build steps",
"type": "row"
},
{
"datasource": {
"type": "postgres",
@ -1423,7 +1410,7 @@
"h": 8,
"w": 8,
"x": 0,
"y": 13
"y": 12
},
"id": 20,
"options": {
@ -1677,7 +1664,7 @@
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ",
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession",
"sql": {
"columns": [
@ -1735,7 +1722,7 @@
"h": 8,
"w": 16,
"x": 8,
"y": 13
"y": 12
},
"id": 21,
"options": {
@ -1981,8 +1968,8 @@
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession",
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_processing",
"sql": {
"columns": [
{
@ -2005,13 +1992,495 @@
"title": "Duration (absolute)",
"type": "bargauge"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 20
},
"id": 27,
"panels": [],
"title": "Tests",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 13,
"x": 0,
"y": 21
},
"id": 29,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n COUNT(tt.id),\n enum.value AS status\nFROM test_tasks AS tt\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (status);\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests count (group by status)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 11,
"x": 13,
"y": 21
},
"id": 31,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n MAX(t.duration) AS \"MAX\",\n MIN(t.duration) AS \"MIN\",\n AVG(t.duration) AS \"AVERAGE\",\n PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY t.duration) as \"MEDIAN\",\n PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY t.duration) as \"95TH PERCENTILE\"\nFROM\n (SELECT \n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\n FROM test_tasks AS tt\n INNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\n INNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n INNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n WHERE b.id = $build_id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\n WHERE b.id = $build_id\n ) AS t;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Tests duration",
"type": "stat"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 27
},
"id": 33,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 8,
"x": 0,
"y": 33
},
"id": 35,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n aenum.value AS arch,\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN build_tasks as bt\n ON tt.build_task_id = bt.id\nINNER JOIN arch_enum as aenum\n ON bt.arch_id = aenum.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (aenum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by arch)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 16,
"x": 8,
"y": 33
},
"id": 37,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
"y": 39
},
"id": 18,
"panels": [
@ -2079,7 +2548,7 @@
"h": 9,
"w": 24,
"x": 0,
"y": 3
"y": 22
},
"id": 2,
"options": {
@ -2286,7 +2755,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 12
"y": 31
},
"id": 25,
"options": {
@ -2367,7 +2836,374 @@
"type": "table"
}
],
"title": "Build tasks",
"title": "Details: build tasks",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 40
},
"id": 39,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 322
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 93
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 62
}
]
},
{
"matcher": {
"id": "byName",
"options": "build task id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}"
}
]
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 1
},
"id": 41,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.build_task_id as \"build task id\",\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n WHERE b.id = $build_id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE b.id = $build_id;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 8
},
"id": 43,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tss.test_task_id AS \"test task id\", \n tt.package_fullname AS \"package name\",\n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: tests",
"type": "row"
}
],
@ -2406,6 +3242,6 @@
"timezone": "",
"title": "Build details",
"uid": "dmVtrz-4k",
"version": 21,
"version": 34,
"weekStart": ""
}

602
grafana-dashbords/Build task details.json → build_analytics/grafana-dashbords/Build task details.json
View File

@ -35,6 +35,12 @@
"name": "Grafana",
"version": "9.3.2"
},
{
"type": "panel",
"id": "piechart",
"name": "Pie chart",
"version": ""
},
{
"type": "datasource",
"id": "postgres",
@ -77,6 +83,17 @@
"links": [],
"liveNow": false,
"panels": [
{
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 28,
"title": "Build task",
"type": "row"
},
{
"datasource": {
"type": "postgres",
@ -143,7 +160,7 @@
"h": 3,
"w": 24,
"x": 0,
"y": 0
"y": 1
},
"id": 5,
"options": {
@ -196,7 +213,7 @@
"urlPath": ""
}
],
"title": "Task summary",
"title": "Build task",
"transformations": [
{
"id": "convertFieldType",
@ -259,10 +276,10 @@
]
},
"gridPos": {
"h": 9,
"h": 8,
"w": 24,
"x": 0,
"y": 3
"y": 4
},
"id": 12,
"options": {
@ -629,14 +646,219 @@
"type": "bargauge"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 12
},
"id": 8,
"id": 22,
"panels": [],
"title": "Test tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 13
},
"id": 24,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 13,
"x": 11,
"y": 13
},
"id": 18,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nWHERE tt.build_task_id = $build_task\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
},
"id": 20,
"panels": [
{
"datasource": {
@ -650,7 +872,7 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
@ -712,7 +934,7 @@
"h": 5,
"w": 24,
"x": 0,
"y": 13
"y": 22
},
"id": 2,
"options": {
@ -750,7 +972,7 @@
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"rawSql": "SELECT \n enum.value as step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
@ -805,7 +1027,7 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
@ -852,10 +1074,10 @@
]
},
"gridPos": {
"h": 11,
"h": 8,
"w": 24,
"x": 0,
"y": 18
"y": 27
},
"id": 3,
"options": {
@ -893,7 +1115,7 @@
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"rawSql": "SELECT enum.value AS step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
@ -937,7 +1159,357 @@
"type": "table"
}
],
"title": "Details",
"title": "Details: build task",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 22
},
"id": 30,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 389
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 72
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 119
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 23
},
"id": 26,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "id"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.build_task_id = $build_task\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 28
},
"id": 16,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: test tasks",
"type": "row"
}
],
@ -975,6 +1547,6 @@
"timezone": "",
"title": "Build task details",
"uid": "vtQClqxVk",
"version": 37,
"version": 61,
"weekStart": ""
}

516
build_analytics/grafana-dashbords/Test task details.json Normal file
View File

@ -0,0 +1,516 @@
{
"__inputs": [
{
"name": "DS_POSTGRESQL",
"label": "PostgreSQL",
"description": "",
"type": "datasource",
"pluginId": "postgres",
"pluginName": "PostgreSQL"
}
],
"__elements": {},
"__requires": [
{
"type": "panel",
"id": "bargauge",
"name": "Bar gauge",
"version": ""
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.3.2"
},
{
"type": "datasource",
"id": "postgres",
"name": "PostgreSQL",
"version": "1.0.0"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 253
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
}
]
},
"gridPos": {
"h": 3,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.id = $id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.id = $id;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Task info",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 3
},
"id": 8,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 11
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"hide": 0,
"includeAll": false,
"label": "Test task id",
"multi": false,
"name": "id",
"options": [],
"query": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Test task details",
"uid": "8nFXlkB4z",
"version": 8,
"weekStart": ""
}

4
build_analytics/migrations/1.sql
View File

@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS builds_finished_at
ON builds(finished_at);
-- build_taks_enum
-- build_tasks_enum
CREATE TABLE IF NOT EXISTS build_task_status_enum(
id INTEGER PRIMARY KEY,
value VARCHAR(15)
@ -53,7 +53,7 @@ CREATE TABLE web_node_stats_enum (
);
INSERT INTO web_node_stats_enum (id, value)
VALUEs
VALUES
(0, 'build_done'),
(1, 'logs_processing'),
(2, 'packages_processing');

77
build_analytics/migrations/3.sql Normal file
View File

@ -0,0 +1,77 @@
BEGIN;
-- test_tasks_status_enum
CREATE TABLE test_tasks_status_enum(
id INTEGER PRIMARY KEY,
value VARCHAR(15)
);
INSERT INTO test_tasks_status_enum (id, value)
VALUES
(1, 'created'),
(2, 'started'),
(3, 'completed'),
(4, 'failed');
-- test_tasks
CREATE TABLE test_tasks (
id INTEGER PRIMARY KEY,
build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
revision INTEGER,
status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
package_fullname VARCHAR(100),
started_at DOUBLE PRECISION
);
CREATE INDEX test_tasks_build_task_id
ON test_tasks(build_task_id);
CREATE INDEX test_tasks_build_status_id
ON test_tasks(status_id);
CREATE INDEX test_tasks_package_fullname
ON test_tasks(package_fullname);
-- test_steps_enum
CREATE TABLE test_steps_enum (
id INTEGER PRIMARY KEY,
value VARCHAR(50)
);
INSERT INTO test_steps_enum (id, value)
VALUES
(0, 'install_package'),
(1, 'stop_environment'),
(2, 'initial_provision'),
(3, 'start_environment'),
(4, 'uninstall_package'),
(5, 'initialize_terraform'),
(6, 'package_integrity_tests'),
(7, 'stop_environment');
-- test_steps_stats
CREATE TABLE test_steps_stats(
test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
stat_name_id INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
start_ts DOUBLE PRECISION,
finish_ts DOUBLE PRECISION
);
ALTER TABLE test_steps_stats
ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
CREATE INDEX test_steps_stats_start_ts
ON test_steps_stats(start_ts);
CREATE INDEX test_steps_stats_end_ts
ON test_steps_stats(end_ts);
UPDATE schema_version
SET version = 3;
COMMIT;

6
build_analytics/releases.txt
View File

@ -7,4 +7,8 @@ First version
- Added metrics for build steps
0.2.1 (2023-03-15)
- Added canceled Build task status
- Added canceled Build task status
0.3.0 (IN PROGRESS)
- Added test tasks stats
- New config parameter: oldest_to_update_days