kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

ALBS-1043 #2

Merged
kzhukov merged 7 commits from ALBS-1043 into main 2023-03-22 10:35:04 +00:00
Conversation 0 Commits 7 Files Changed 20 +5184 -1948
20 changed files with 5184 additions and 1948 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
build_analytics/build_analytics/api_client.py
View File

@ -1,16 +1,19 @@
from datetime import datetime from datetime import datetime
import logging import logging
from urllib.parse import urljoin from urllib.parse import urljoin
from typing import Dict, List from typing import Dict, List, Any
import requests import requests
from .models.build import Build from .models.build import Build
from .models.build_task import BuildTask from .models.build_task import BuildTask
from .models.build_node_stats import BuildNodeStats from .models.build_node_stats import BuildNodeStats
from .models.build_stat import BuildStat from .models.build_stat import BuildStat
from .models.web_node_stats import WebNodeStats from .models.web_node_stats import WebNodeStats
from .models.test_task import TestTask
from .models.test_steps_stats import TestStepsStats
from .models.test_step_stat import TestStepStat
TZ_OFFSET = '+00:00' TZ_OFFSET = '+00:00'
@ -138,3 +141,63 @@ class APIclient():
'build_tasks': build_tasks} 'build_tasks': build_tasks}
return Build(**params) return Build(**params)
def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
ep = f'/api/v1/tests/{build_task_id}/latest'
url = urljoin(self.api_root, ep)
headers = {'accept': 'application/json'}
response = requests.get(
url, headers=headers, timeout=self.timeout)
response.raise_for_status()
return self.__parse_test_tasks(response.json(), build_task_id)
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
build_task_id: int,
started_at: str = None) -> List[TestTask]:
result: List[TestTask] = []
for task in raw_tasks:
if task['alts_response']:
try:
started_raw = task['alts_response']['stats']['started_at']
except KeyError:
started_at = None
else:
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
try:
stats_raw = task['alts_response']['stats']
except KeyError:
steps_stats = None
else:
steps_stats = self.__parse_test_steps_stats(stats_raw)
else:
started_at = None
steps_stats = None
params = {
'id': task['id'],
'build_task_id': build_task_id,
'revision': task['revision'],
'status': task['status'],
'package_fullname': '_'.join([task['package_name'],
task['package_version'],
task['package_release']]),
'started_at': started_at,
'steps_stats': steps_stats
}
result.append(TestTask(**params))
return result
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
teast_steps_params = {}
for field_name in TestStepsStats.__fields__.keys():
try:
p = stats_raw[field_name]
except KeyError:
continue
# there are must be a better way
for k in ['start_ts', 'finish_ts']:
if k in p:
p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params)

20
build_analytics/build_analytics/const.py
View File

@ -3,7 +3,7 @@
from enum import IntEnum from enum import IntEnum
# supported schema version # supported schema version
DB_SCHEMA_VER = 2 DB_SCHEMA_VER = 3
# ENUMS # ENUMS
@ -41,3 +41,21 @@ class BuildNodeStatsEnum(IntEnum):
build_node_task = 6 build_node_task = 6
cas_notarize_artifacts = 7 cas_notarize_artifacts = 7
cas_source_authenticate = 8 cas_source_authenticate = 8
class TestTaskStatusEnum(IntEnum):
created = 1
started = 2
completed = 3
failed = 4
class TestStepEnum(IntEnum):
install_package = 0
stop_enviroment = 1
initial_provision = 2
start_environment = 3
uninstall_package = 4
initialize_terraform = 5
package_integrity_tests = 6
stop_environment = 7

123
build_analytics/build_analytics/db.py
View File

@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
from .models.build_node_stat_db import BuildNodeStatDB from .models.build_node_stat_db import BuildNodeStatDB
from .models.db_config import DbConfig from .models.db_config import DbConfig
from .models.web_node_stat_db import WebNodeStatDB from .models.web_node_stat_db import WebNodeStatDB
from .models.test_task_db import TestTaskDB
class DB(): class DB():
@ -25,6 +26,17 @@ class DB():
def __del__(self): def __del__(self):
self.close_conn() self.close_conn()
def build_exists(self, build_id: int) -> bool:
sql = f'''
SELECT COUNT(id)
FROM builds
WHERE id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (build_id,))
val = int(cur.fetchone()[0])
return val == 1
def insert_build(self, build: BuildDB): def insert_build(self, build: BuildDB):
sql = ''' sql = '''
INSERT INTO builds(id, url, created_at, finished_at) INSERT INTO builds(id, url, created_at, finished_at)
@ -99,7 +111,7 @@ class DB():
self.__conn.commit() self.__conn.commit()
return cur.rowcount return cur.rowcount
def get_unfinished_builds(self) -> Dict[int, Dict[int, int]]: def get_unfinished_builds(self, not_before: datetime) -> Dict[int, Dict[int, int]]:
""" """
Getting list of unfinished builds and build_tasks Getting list of unfinished builds and build_tasks
Dict[build_id, Dict[build_task_id, task_status_id]] Dict[build_id, Dict[build_task_id, task_status_id]]
@ -107,9 +119,10 @@ class DB():
res: Dict[int, Dict[int, int]] = {} res: Dict[int, Dict[int, int]] = {}
# getting unfinished builds # getting unfinished builds
sql = 'SELECT id FROM builds where finished_at is NULL;' sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
cur = self.__conn.cursor() cur = self.__conn.cursor()
cur.execute(sql) cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
for row in cur.fetchall(): for row in cur.fetchall():
res[row[0]] = {} res[row[0]] = {}
@ -159,9 +172,10 @@ class DB():
for stat in web_node_stats: for stat in web_node_stats:
logging.debug( logging.debug(
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id) 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id, if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id, stat_name_id=stat.stat_name_id,
table_name='web_node_stats'): table_name='web_node_stats',
column_name='build_task_id'):
sql = ''' sql = '''
UPDATE web_node_stats UPDATE web_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -183,9 +197,10 @@ class DB():
for stat in build_node_stats: for stat in build_node_stats:
logging.debug( logging.debug(
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id) 'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
if self.stat_exists(build_task_id=stat.build_task_id, if self.stat_exists(task_id=stat.build_task_id,
stat_name_id=stat.stat_name_id, stat_name_id=stat.stat_name_id,
table_name='build_node_stats'): table_name='build_node_stats',
column_name='build_task_id'):
sql = ''' sql = '''
UPDATE build_node_stats UPDATE build_node_stats
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
@ -219,13 +234,99 @@ class DB():
return None return None
return int(val[0]) return int(val[0])
def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool: def stat_exists(self, task_id: int, stat_name_id: int, table_name: str, column_name: str) -> bool:
sql = f''' sql = f'''
SELECT COUNT(build_task_id) SELECT COUNT({column_name})
FROM {table_name} FROM {table_name}
WHERE build_task_id = %s AND stat_name_id = %s; WHERE {column_name} = %s AND stat_name_id = %s;
''' '''
cur = self.__conn.cursor() cur = self.__conn.cursor()
cur.execute(sql, (build_task_id, stat_name_id)) cur.execute(sql, (task_id, stat_name_id))
val = int(cur.fetchone()[0]) val = int(cur.fetchone()[0])
return val == 1 return val == 1
def insert_test_task(self, task: TestTaskDB):
cur = self.__conn.cursor()
# inserting test task itself
sql = '''
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
VALUES
(%s, %s, %s, %s, %s, %s);
'''
cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
task.package_fullname, task.started_at))
if task.steps_stats:
# inserting test steps stats
for ss in task.steps_stats:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
VALUES
(%s, %s, %s, %s);
'''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.finish_ts))
# commiting changes
self.__conn.commit()
def get_build_tasks_for_unfinished_tests(self, not_before: datetime) -> List[int]:
'''
getting build tasks id of unfinished test tasks
'''
cur = self.__conn.cursor()
sql = '''
SELECT DISTINCT bt.id
FROM build_tasks as bt
INNER JOIN test_tasks AS tt
ON bt.id = tt.build_task_id
WHERE tt.status_id < 3 AND bt.started_at > %s;
'''
cur.execute(sql, (not_before.timestamp(),))
logging.debug('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()]
return result
def update_test_tasks(self, test_tasks: List[TestTaskDB]):
cur = self.__conn.cursor()
# test tasks
for task in test_tasks:
sql = '''
UPDATE test_tasks
SET revision = %s,
status_id = %s,
started_at = %s
WHERE id = %s;
'''
cur.execute(sql, (task.revision, task.status_id,
task.started_at, task.id))
assert cur.rowcount == 1
# test step
if not task.steps_stats:
continue
for s in task.steps_stats:
logging.info('test_task_id %s, stat_name_id %s',
s.test_task_id, s.stat_name_id)
if self.stat_exists(s.test_task_id,
s.stat_name_id,
'test_steps_stats',
'test_task_id'):
sql = '''
UPDATE test_steps_stats
SET start_ts = %s,
finish_ts = %s
WHERE test_task_id = %s AND stat_name_id = %s;
'''
cur.execute(sql, (s.start_ts, s.finish_ts,
s.test_task_id, s.stat_name_id))
assert cur.rowcount == 1
else:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
VALUES (%s, %s, %s, %s);
'''
cur.execute(sql, (s.test_task_id, s.stat_name_id,
s.start_ts, s.finish_ts))
# commiting changes
self.__conn.commit()

61
build_analytics/build_analytics/extractor/extractor.py
View File

@ -1,19 +1,18 @@
# pylint: disable=relative-beyond-top-level # pylint: disable=relative-beyond-top-level
import logging import logging
from typing import List, Dict from typing import Dict, List
from ..models.extractor_config import ExtractorConfig
from ..const import BuildTaskEnum
from ..models.build import BuildTask
from ..db import DB
from ..api_client import APIclient from ..api_client import APIclient
from ..const import BuildTaskEnum
from ..db import DB
from ..models.build import BuildTask
from ..models.extractor_config import ExtractorConfig
class Extractor: class Extractor:
def __init__(self, config: ExtractorConfig, api: APIclient, db: DB): def __init__(self, config: ExtractorConfig, api: APIclient, db: DB):
self.start_from = config.start_from self.config = config
self.oldest_build_age = config.oldest_build_age
self.api = api self.api = api
self.db = db self.db = db
@ -22,7 +21,7 @@ class Extractor:
page_num = 1 page_num = 1
last_build_id = self.db.get_latest_build_id() last_build_id = self.db.get_latest_build_id()
if not last_build_id: if not last_build_id:
last_build_id = self.start_from - 1 last_build_id = self.config.start_from - 1
logging.info("last_build_id: %s", last_build_id) logging.info("last_build_id: %s", last_build_id)
stop = False stop = False
@ -31,12 +30,16 @@ class Extractor:
for build in self.api.get_builds(page_num): for build in self.api.get_builds(page_num):
# check if we shoud stop processing build # check if we shoud stop processing build
if build.id <= last_build_id or \ if build.id <= last_build_id or \
build.created_at <= self.oldest_build_age: build.created_at <= self.config.oldest_build_age:
stop = True stop = True
break break
# some builds could move from one page to another
if self.db.build_exists(build_id=build.id):
continue
# inserting build build tasks and build tasks statistics # inserting build build tasks and build tasks statistics
logging.info("inserting %s", build.id) logging.info('inserting %s', build.id)
try: try:
self.db.insert_build(build.as_db_model()) self.db.insert_build(build.as_db_model())
except Exception as error: # pylint: disable=broad-except except Exception as error: # pylint: disable=broad-except
@ -45,6 +48,8 @@ class Extractor:
continue continue
for build_task in build.build_tasks: for build_task in build.build_tasks:
logging.info('build %s: inserting build task %s',
build.id, build_task.id)
try: try:
self.db.insert_buildtask(build_task.as_db_model(), self.db.insert_buildtask(build_task.as_db_model(),
build_task.web_node_stats.as_db_model( build_task.web_node_stats.as_db_model(
@ -52,16 +57,25 @@ class Extractor:
build_task.build_node_stats.as_db_model( build_task.build_node_stats.as_db_model(
build_task.id)) build_task.id))
except Exception as error: # pylint: disable=broad-except except Exception as error: # pylint: disable=broad-except
logging.error('failed to insert build task %d: %s', logging.error('build %s: failed to insert build task %d: %s',
build_task.id, error, exc_info=True) build.id, build_task.id, error, exc_info=True)
logging.info(
'getting test tasks for build task %s', build_task.id)
test_tasks = self.api.get_test_tasks(build_task.id)
logging.info('received %d tests tasks', len(test_tasks))
for t in test_tasks:
logging.info(
'build task %s: inserting test task %s', build_task.id, t.id)
self.db.insert_test_task(t.as_db_model())
build_count += 1 build_count += 1
page_num += 1 page_num += 1
return build_count return build_count
def build_cleanup(self): def build_cleanup(self):
logging.info('Removing all buidls older then %s', logging.info('Removing all buidls older then %s',
self.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S")) self.config.oldest_build_age.strftime("%m/%d/%Y, %H:%M:%S"))
removed_count = self.db.cleanup_builds(self.oldest_build_age) removed_count = self.db.cleanup_builds(self.config.oldest_build_age)
logging.info('removed %d entries', removed_count) logging.info('removed %d entries', removed_count)
def __update_build_tasks(self, build_tasks: List[BuildTask], def __update_build_tasks(self, build_tasks: List[BuildTask],
@ -92,7 +106,8 @@ class Extractor:
def update_builds(self): def update_builds(self):
logging.info('Getting list of tasks from DB') logging.info('Getting list of tasks from DB')
unfinished_tasks = self.db.get_unfinished_builds() unfinished_tasks = self.db.get_unfinished_builds(
self.config.oldest_to_update)
for build_id, build_tasks_db in unfinished_tasks.items(): for build_id, build_tasks_db in unfinished_tasks.items():
try: try:
logging.info('Getting status of build %d', build_id) logging.info('Getting status of build %d', build_id)
@ -114,3 +129,19 @@ class Extractor:
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logging.error("Cant process build %d: %s", logging.error("Cant process build %d: %s",
build_id, err, exc_info=True) build_id, err, exc_info=True)
def updating_test_tasks(self):
logging.info('getting build task ids of unfinished tests')
build_task_ids = self.db.get_build_tasks_for_unfinished_tests(
self.config.oldest_to_update)
for build_task_id in build_task_ids:
try:
logging.info('getting tests for build task %s', build_task_id)
tasks_api = self.api.get_test_tasks(build_task_id)
logging.info('updating test tasks')
tasks_db = [t.as_db_model() for t in tasks_api]
self.db.update_test_tasks(tasks_db)
except Exception as err: # pylint: disable=broad-except
logging.error(
'failed to update tests for %d build task: %s',
build_task_id, err, exc_info=True)

48
build_analytics/build_analytics/extractor/start.py
View File

@ -15,20 +15,6 @@ from ..models.extractor_config import ExtractorConfig
from ..models.db_config import DbConfig from ..models.db_config import DbConfig
def __get_oldest_build_age(config: dict) -> datetime:
oldest_build_age = datetime.now().astimezone() \
- timedelta(days=config['data_store_days'])
return oldest_build_age
def __get_db_config(config: dict) -> DbConfig:
return DbConfig(name=config['db_name'],
port=int(config['db_port']),
host=config['db_host'],
username=config['db_username'],
password=config['db_password'])
def __get_config(yml_path: str) -> ExtractorConfig: def __get_config(yml_path: str) -> ExtractorConfig:
""" """
get_config loads yml file and generates instance get_config loads yml file and generates instance
@ -37,8 +23,18 @@ def __get_config(yml_path: str) -> ExtractorConfig:
raw = yaml.safe_load(flr) raw = yaml.safe_load(flr)
# adding new attrs # adding new attrs
raw['oldest_build_age'] = __get_oldest_build_age(raw) raw['oldest_build_age'] = datetime.now().astimezone() \
raw['db_config'] = __get_db_config(raw) - timedelta(days=raw['data_store_days'])
raw['db_config'] = DbConfig(name=raw['db_name'],
port=int(raw['db_port']),
host=raw['db_host'],
username=raw['db_username'],
password=raw['db_password'])
if 'oldest_to_update_days' in raw:
raw['oldest_to_update_days'] = datetime.now().astimezone() \
- timedelta(days=raw['oldest_to_update_days'])
return ExtractorConfig(**raw) return ExtractorConfig(**raw)
@ -83,21 +79,29 @@ def start(yml_path: str):
logging.info( logging.info(
'Build extaction was finished. %d builds were inserted', inserted_count) 'Build extaction was finished. %d builds were inserted', inserted_count)
logging.info('Starting old builds removal') logging.info('starting old builds removal')
try: try:
extractor.build_cleanup() extractor.build_cleanup()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logging.critical("Unhandled exception %s", err, exc_info=True) logging.critical("unhandled exception %s", err, exc_info=True)
else: else:
logging.info('Cleanup finished') logging.info('cleanup finished')
logging.info('Updating statuses of unfinished build tasks') logging.info('updating statuses of unfinished build tasks')
try: try:
extractor.update_builds() extractor.update_builds()
except Exception as err: # pylint: disable=broad-except except Exception as err: # pylint: disable=broad-except
logging.critical("Unhandled exception %s", err, exc_info=True) logging.critical("unhandled exception %s", err, exc_info=True)
else: else:
logging.info('Update finished') logging.info('update finished')
logging.info('updating statuses of unfinished test tasks')
try:
extractor.updating_test_tasks()
except Exception as err: # pylint: disable=broad-except
logging.critical("unhandled exception %s", err, exc_info=True)
else:
logging.info('test tasks were updated')
extractor.db.close_conn() extractor.db.close_conn()
logging.info("Extraction was finished") logging.info("Extraction was finished")

6
build_analytics/build_analytics/models/extractor_config.py
View File

@ -1,4 +1,4 @@
from datetime import datetime from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
from pydantic import HttpUrl, Field, BaseModel # pylint: disable=no-name-in-module from pydantic import HttpUrl, Field, BaseModel # pylint: disable=no-name-in-module
@ -11,6 +11,7 @@ LOG_FILE_DEFAULT = '/tmp/extractor.log'
API_DEFAULT = 30 API_DEFAULT = 30
SCRAPE_INTERVAL_DEFAULT = 3600 SCRAPE_INTERVAL_DEFAULT = 3600
START_FROM_DEFAULT = 5808 START_FROM_DEFAULT = 5808
OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=7)
class ExtractorConfig(BaseModel): class ExtractorConfig(BaseModel):
@ -32,3 +33,6 @@ class ExtractorConfig(BaseModel):
default=SCRAPE_INTERVAL_DEFAULT) default=SCRAPE_INTERVAL_DEFAULT)
start_from: int = Field(description='build id to start populating empty db with', start_from: int = Field(description='build id to start populating empty db with',
default=START_FROM_DEFAULT) default=START_FROM_DEFAULT)
oldest_to_update: datetime = \
Field(description='oldest unfinished object (build/task/step...) that we will try to update',
default=OLDEST_TO_UPDATE_DEFAULT)

2
build_analytics/build_analytics/models/sign_task_db.py
View File

@ -1,4 +1,4 @@
from pydantic import BaseModel from pydantic import BaseModel # pylint: disable=no-name-in-module
class SignTaskDB(BaseModel): class SignTaskDB(BaseModel):

9
build_analytics/build_analytics/models/test_step_stat.py Normal file
View File

@ -0,0 +1,9 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
class TestStepStat(BaseModel):
start_ts: Optional[datetime] = None
finish_ts: Optional[datetime] = None

9
build_analytics/build_analytics/models/test_step_stat_db.py Normal file
View File

@ -0,0 +1,9 @@
from pydantic import BaseModel # pylint: disable=no-name-in-module
from typing import Optional
class TestStepStatDB(BaseModel):
test_task_id: int
stat_name_id: int
start_ts: Optional[float] = None
finish_ts: Optional[float] = None

36
build_analytics/build_analytics/models/test_steps_stats.py Normal file
View File

@ -0,0 +1,36 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from ..const import TestStepEnum
from .test_step_stat import TestStepStat
from .test_step_stat_db import TestStepStatDB
class TestStepsStats(BaseModel):
install_package: Optional[TestStepStat] = None
stop_environment: Optional[TestStepStat] = None
initial_provision: Optional[TestStepStat] = None
start_environment: Optional[TestStepStat] = None
uninstall_package: Optional[TestStepStat] = None
initialize_terraform: Optional[TestStepStat] = None
package_integrity_tests: Optional[TestStepStat] = None
def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
result = []
for field_name in self.__fields__.keys():
stats: TestStepStat = getattr(self, field_name)
if not stats:
continue
start_ts = stats.start_ts.timestamp() \
if stats.start_ts else None
finish_ts = stats.finish_ts.timestamp() \
if stats.finish_ts else None
stat_name_id = TestStepEnum[field_name].value
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
stat_name_id=stat_name_id,
start_ts=start_ts,
finish_ts=finish_ts)
result.append(test_step_stat_db)
return result

31
build_analytics/build_analytics/models/test_task.py Normal file
View File

@ -0,0 +1,31 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_task_db import TestTaskDB
from .test_steps_stats import TestStepsStats
class TestTask(BaseModel):
id: int
build_task_id: int
revision: int
status: int
package_fullname: str
started_at: Optional[datetime] = None
steps_stats: Optional[TestStepsStats] = None
def as_db_model(self) -> TestTaskDB:
started_at = self.started_at.timestamp() \
if self.started_at else None
params = {
'id': self.id,
'build_task_id': self.build_task_id,
'revision': self.revision,
'status_id': self.status,
'package_fullname': self.package_fullname,
'started_at': started_at,
'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
}
return TestTaskDB(**params)

17
build_analytics/build_analytics/models/test_task_db.py Normal file
View File

@ -0,0 +1,17 @@
from typing import List, Optional
from pydantic import BaseModel # pylint: disable=no-name-in-module
from .test_step_stat_db import TestStepStatDB
class TestTaskDB(BaseModel):
"""
Test task as it received from/sent to database
"""
id: int
build_task_id: int
revision: int
status_id: int
package_fullname: str
started_at: Optional[float] = None
steps_stats: Optional[List[TestStepStatDB]] = None

8
build_analytics/config_default.yml
View File

@ -61,3 +61,11 @@ scrape_interval: 3600
# required: false # required: false
# default: 5808 (first build with correct metrics) # default: 5808 (first build with correct metrics)
start_from: start_from:
# oldest_to_update
# oldest (in days) unfinished object (build/task/step...) that we will try to update
# required: false
# default: 7
oldest_to_update_days: 7

4606
grafana-dashbords/Build analytics.json → build_analytics/grafana-dashbords/Build analytics.json
View File

File diff suppressed because it is too large Load Diff

884
grafana-dashbords/Build details.json → build_analytics/grafana-dashbords/Build details.json
View File

@ -87,7 +87,7 @@
}, },
"id": 16, "id": 16,
"panels": [], "panels": [],
"title": "Summary", "title": "Builds",
"type": "row" "type": "row"
}, },
{ {
@ -1261,19 +1261,6 @@
], ],
"type": "stat" "type": "stat"
}, },
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 12
},
"id": 23,
"panels": [],
"title": "Build steps",
"type": "row"
},
{ {
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
@ -1423,7 +1410,7 @@
"h": 8, "h": 8,
"w": 8, "w": 8,
"x": 0, "x": 0,
"y": 13 "y": 12
}, },
"id": 20, "id": 20,
"options": { "options": {
@ -1677,7 +1664,7 @@
"format": "table", "format": "table",
"hide": false, "hide": false,
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", "rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession", "refId": "packages_procession",
"sql": { "sql": {
"columns": [ "columns": [
@ -1735,7 +1722,7 @@
"h": 8, "h": 8,
"w": 16, "w": 16,
"x": 8, "x": 8,
"y": 13 "y": 12
}, },
"id": 21, "id": 21,
"options": { "options": {
@ -1981,8 +1968,8 @@
"format": "table", "format": "table",
"hide": false, "hide": false,
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", "rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession", "refId": "packages_processing",
"sql": { "sql": {
"columns": [ "columns": [
{ {
@ -2005,13 +1992,495 @@
"title": "Duration (absolute)", "title": "Duration (absolute)",
"type": "bargauge" "type": "bargauge"
}, },
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 20
},
"id": 27,
"panels": [],
"title": "Tests",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 13,
"x": 0,
"y": 21
},
"id": 29,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n COUNT(tt.id),\n enum.value AS status\nFROM test_tasks AS tt\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (status);\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests count (group by status)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 11,
"x": 13,
"y": 21
},
"id": 31,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n MAX(t.duration) AS \"MAX\",\n MIN(t.duration) AS \"MIN\",\n AVG(t.duration) AS \"AVERAGE\",\n PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY t.duration) as \"MEDIAN\",\n PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY t.duration) as \"95TH PERCENTILE\"\nFROM\n (SELECT \n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\n FROM test_tasks AS tt\n INNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\n INNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n INNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n WHERE b.id = $build_id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\n WHERE b.id = $build_id\n ) AS t;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Tests duration",
"type": "stat"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 27
},
"id": 33,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 8,
"x": 0,
"y": 33
},
"id": 35,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n aenum.value AS arch,\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN build_tasks as bt\n ON tt.build_task_id = bt.id\nINNER JOIN arch_enum as aenum\n ON bt.arch_id = aenum.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (aenum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by arch)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 16,
"x": 8,
"y": 33
},
"id": 37,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{ {
"collapsed": true, "collapsed": true,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 21 "y": 39
}, },
"id": 18, "id": 18,
"panels": [ "panels": [
@ -2079,7 +2548,7 @@
"h": 9, "h": 9,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 3 "y": 22
}, },
"id": 2, "id": 2,
"options": { "options": {
@ -2286,7 +2755,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 12 "y": 31
}, },
"id": 25, "id": 25,
"options": { "options": {
@ -2367,7 +2836,374 @@
"type": "table" "type": "table"
} }
], ],
"title": "Build tasks", "title": "Details: build tasks",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 40
},
"id": 39,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 322
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 93
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 62
}
]
},
{
"matcher": {
"id": "byName",
"options": "build task id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}"
}
]
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 1
},
"id": 41,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.build_task_id as \"build task id\",\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n INNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\n INNER JOIN builds AS b\n ON bt.build_id = b.id\n WHERE b.id = $build_id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE b.id = $build_id;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 8
},
"id": 43,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tss.test_task_id AS \"test task id\", \n tt.package_fullname AS \"package name\",\n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nINNER JOIN build_tasks AS bt\n ON tt.build_task_id = bt.id\nINNER JOIN builds AS b\n ON bt.build_id = b.id\nWHERE b.id = $build_id\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: tests",
"type": "row" "type": "row"
} }
], ],
@ -2406,6 +3242,6 @@
"timezone": "", "timezone": "",
"title": "Build details", "title": "Build details",
"uid": "dmVtrz-4k", "uid": "dmVtrz-4k",
"version": 21, "version": 34,
"weekStart": "" "weekStart": ""
} }

602
grafana-dashbords/Build task details.json → build_analytics/grafana-dashbords/Build task details.json
View File

@ -35,6 +35,12 @@
"name": "Grafana", "name": "Grafana",
"version": "9.3.2" "version": "9.3.2"
}, },
{
"type": "panel",
"id": "piechart",
"name": "Pie chart",
"version": ""
},
{ {
"type": "datasource", "type": "datasource",
"id": "postgres", "id": "postgres",
@ -77,6 +83,17 @@
"links": [], "links": [],
"liveNow": false, "liveNow": false,
"panels": [ "panels": [
{
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 28,
"title": "Build task",
"type": "row"
},
{ {
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
@ -143,7 +160,7 @@
"h": 3, "h": 3,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 0 "y": 1
}, },
"id": 5, "id": 5,
"options": { "options": {
@ -196,7 +213,7 @@
"urlPath": "" "urlPath": ""
} }
], ],
"title": "Task summary", "title": "Build task",
"transformations": [ "transformations": [
{ {
"id": "convertFieldType", "id": "convertFieldType",
@ -259,10 +276,10 @@
] ]
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 8,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 3 "y": 4
}, },
"id": 12, "id": 12,
"options": { "options": {
@ -629,14 +646,219 @@
"type": "bargauge" "type": "bargauge"
}, },
{ {
"collapsed": true, "collapsed": false,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 12 "y": 12
}, },
"id": 8, "id": 22,
"panels": [],
"title": "Test tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 13
},
"id": 24,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 13,
"x": 11,
"y": 13
},
"id": 18,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nWHERE tt.build_task_id = $build_task\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
},
"id": 20,
"panels": [ "panels": [
{ {
"datasource": { "datasource": {
@ -650,7 +872,7 @@
"mode": "thresholds" "mode": "thresholds"
}, },
"custom": { "custom": {
"align": "auto", "align": "left",
"displayMode": "auto", "displayMode": "auto",
"filterable": true, "filterable": true,
"inspect": true "inspect": true
@ -712,7 +934,7 @@
"h": 5, "h": 5,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 13 "y": 22
}, },
"id": 2, "id": 2,
"options": { "options": {
@ -750,7 +972,7 @@
"method": "GET", "method": "GET",
"queryParams": "", "queryParams": "",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", "rawSql": "SELECT \n enum.value as step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats", "refId": "web node stats",
"sql": { "sql": {
"columns": [ "columns": [
@ -805,7 +1027,7 @@
"mode": "thresholds" "mode": "thresholds"
}, },
"custom": { "custom": {
"align": "auto", "align": "left",
"displayMode": "auto", "displayMode": "auto",
"filterable": true, "filterable": true,
"inspect": true "inspect": true
@ -852,10 +1074,10 @@
] ]
}, },
"gridPos": { "gridPos": {
"h": 11, "h": 8,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 18 "y": 27
}, },
"id": 3, "id": 3,
"options": { "options": {
@ -893,7 +1115,7 @@
"method": "GET", "method": "GET",
"queryParams": "", "queryParams": "",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", "rawSql": "SELECT enum.value AS step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats", "refId": "web node stats",
"sql": { "sql": {
"columns": [ "columns": [
@ -937,7 +1159,357 @@
"type": "table" "type": "table"
} }
], ],
"title": "Details", "title": "Details: build task",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 22
},
"id": 30,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 389
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 72
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 119
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 23
},
"id": 26,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "id"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.build_task_id = $build_task\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 28
},
"id": 16,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: test tasks",
"type": "row" "type": "row"
} }
], ],
@ -975,6 +1547,6 @@
"timezone": "", "timezone": "",
"title": "Build task details", "title": "Build task details",
"uid": "vtQClqxVk", "uid": "vtQClqxVk",
"version": 37, "version": 61,
"weekStart": "" "weekStart": ""
} }

516
build_analytics/grafana-dashbords/Test task details.json Normal file
View File

@ -0,0 +1,516 @@
{
"__inputs": [
{
"name": "DS_POSTGRESQL",
"label": "PostgreSQL",
"description": "",
"type": "datasource",
"pluginId": "postgres",
"pluginName": "PostgreSQL"
}
],
"__elements": {},
"__requires": [
{
"type": "panel",
"id": "bargauge",
"name": "Bar gauge",
"version": ""
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.3.2"
},
{
"type": "datasource",
"id": "postgres",
"name": "PostgreSQL",
"version": "1.0.0"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 253
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
}
]
},
"gridPos": {
"h": 3,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.id = $id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.id = $id;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Task info",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 3
},
"id": 8,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 11
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"hide": 0,
"includeAll": false,
"label": "Test task id",
"multi": false,
"name": "id",
"options": [],
"query": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Test task details",
"uid": "8nFXlkB4z",
"version": 8,
"weekStart": ""
}

4
build_analytics/migrations/1.sql
View File

@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS builds_finished_at
ON builds(finished_at); ON builds(finished_at);
-- build_taks_enum -- build_tasks_enum
CREATE TABLE IF NOT EXISTS build_task_status_enum( CREATE TABLE IF NOT EXISTS build_task_status_enum(
id INTEGER PRIMARY KEY, id INTEGER PRIMARY KEY,
value VARCHAR(15) value VARCHAR(15)
@ -53,7 +53,7 @@ CREATE TABLE web_node_stats_enum (
); );
INSERT INTO web_node_stats_enum (id, value) INSERT INTO web_node_stats_enum (id, value)
VALUEs VALUES
(0, 'build_done'), (0, 'build_done'),
(1, 'logs_processing'), (1, 'logs_processing'),
(2, 'packages_processing'); (2, 'packages_processing');

77
build_analytics/migrations/3.sql Normal file
View File

@ -0,0 +1,77 @@
BEGIN;
-- test_tasks_status_enum
CREATE TABLE test_tasks_status_enum(
id INTEGER PRIMARY KEY,
value VARCHAR(15)
);
INSERT INTO test_tasks_status_enum (id, value)
VALUES
(1, 'created'),
(2, 'started'),
(3, 'completed'),
(4, 'failed');
-- test_tasks
CREATE TABLE test_tasks (
id INTEGER PRIMARY KEY,
build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
revision INTEGER,
status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
package_fullname VARCHAR(100),
started_at DOUBLE PRECISION
);
CREATE INDEX test_tasks_build_task_id
ON test_tasks(build_task_id);
CREATE INDEX test_tasks_build_status_id
ON test_tasks(status_id);
CREATE INDEX test_tasks_package_fullname
ON test_tasks(package_fullname);
-- test_steps_enum
CREATE TABLE test_steps_enum (
id INTEGER PRIMARY KEY,
value VARCHAR(50)
);
INSERT INTO test_steps_enum (id, value)
VALUES
(0, 'install_package'),
(1, 'stop_environment'),
(2, 'initial_provision'),
(3, 'start_environment'),
(4, 'uninstall_package'),
(5, 'initialize_terraform'),
(6, 'package_integrity_tests'),
(7, 'stop_environment');
-- test_steps_stats
CREATE TABLE test_steps_stats(
test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
stat_name_id INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
start_ts DOUBLE PRECISION,
finish_ts DOUBLE PRECISION
);
ALTER TABLE test_steps_stats
ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
CREATE INDEX test_steps_stats_start_ts
ON test_steps_stats(start_ts);
CREATE INDEX test_steps_stats_end_ts
ON test_steps_stats(end_ts);
UPDATE schema_version
SET version = 3;
COMMIT;

6
build_analytics/releases.txt
View File

@ -7,4 +7,8 @@ First version
- Added metrics for build steps - Added metrics for build steps
0.2.1 (2023-03-15) 0.2.1 (2023-03-15)
- Added canceled Build task status - Added canceled Build task status
0.3.0 (IN PROGRESS)
- Added test tasks stats
- New config parameter: oldest_to_update_days