From d737bae3537895a47e4a1b57258781dc2666331b Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Mon, 6 Mar 2023 19:21:56 +0100 Subject: [PATCH 01/12] added build task stats --- .../build_analytics/models/enums.py | 17 - build_analitycs/releases.txt | 2 - .../__init__.py | 0 .../build_analytics/__init__.py | 0 .../build_analytics/api_client.py | 59 +- .../build_analytics/db.py | 66 +- .../build_analytics/extractor/__jnit__.py | 0 .../build_analytics/extractor/extractor.py | 47 +- .../build_analytics/extractor/start.py | 0 .../build_analytics/models/__init__.py | 0 .../build_analytics/models/build.py | 0 .../build_analytics/models/build_db.py | 0 .../models/build_node_stat_db.py | 12 + .../models/build_node_stats.py | 41 + .../build_analytics/models/build_stat.py | 15 + .../build_analytics/models/build_stat_db.py | 16 + .../build_analytics/models/build_task.py | 6 +- .../build_analytics/models/build_task_db.py | 0 .../build_analytics/models/db_config.py | 0 .../build_analytics/models/enums.py | 37 + .../models/extractor_config.py | 5 +- .../build_analytics/models/sign_task_db.py | 0 .../models/web_node_stat_db.py | 13 + .../build_analytics/models/web_node_stats.py | 35 + .../config_default.yml | 7 +- .../migrations/1.sql | 86 +- build_analytics/releases.txt | 5 + .../requirements.txt | 0 .../run_extractor.py | 0 grafana-dashbords/Build analytics.json | 1509 +++++++++++++++++ grafana-dashbords/albs_analytics.json | 280 --- 31 files changed, 1920 insertions(+), 338 deletions(-) delete mode 100644 build_analitycs/build_analytics/models/enums.py delete mode 100644 build_analitycs/releases.txt rename {build_analitycs => build_analytics}/__init__.py (100%) rename {build_analitycs => build_analytics}/build_analytics/__init__.py (100%) rename {build_analitycs => build_analytics}/build_analytics/api_client.py (54%) rename {build_analitycs => build_analytics}/build_analytics/db.py (62%) rename {build_analitycs => build_analytics}/build_analytics/extractor/__jnit__.py (100%) rename {build_analitycs => build_analytics}/build_analytics/extractor/extractor.py (59%) rename {build_analitycs => build_analytics}/build_analytics/extractor/start.py (100%) rename {build_analitycs => build_analytics}/build_analytics/models/__init__.py (100%) rename {build_analitycs => build_analytics}/build_analytics/models/build.py (100%) rename {build_analitycs => build_analytics}/build_analytics/models/build_db.py (100%) create mode 100644 build_analytics/build_analytics/models/build_node_stat_db.py create mode 100644 build_analytics/build_analytics/models/build_node_stats.py create mode 100644 build_analytics/build_analytics/models/build_stat.py create mode 100644 build_analytics/build_analytics/models/build_stat_db.py rename {build_analitycs => build_analytics}/build_analytics/models/build_task.py (82%) rename {build_analitycs => build_analytics}/build_analytics/models/build_task_db.py (100%) rename {build_analitycs => build_analytics}/build_analytics/models/db_config.py (100%) create mode 100644 build_analytics/build_analytics/models/enums.py rename {build_analitycs => build_analytics}/build_analytics/models/extractor_config.py (82%) rename {build_analitycs => build_analytics}/build_analytics/models/sign_task_db.py (100%) create mode 100644 build_analytics/build_analytics/models/web_node_stat_db.py create mode 100644 build_analytics/build_analytics/models/web_node_stats.py rename {build_analitycs => build_analytics}/config_default.yml (87%) rename build_analitycs/db_schema/postgres.sql => build_analytics/migrations/1.sql (51%) create mode 100644 build_analytics/releases.txt rename {build_analitycs => build_analytics}/requirements.txt (100%) rename {build_analitycs => build_analytics}/run_extractor.py (100%) create mode 100644 grafana-dashbords/Build analytics.json delete mode 100644 grafana-dashbords/albs_analytics.json diff --git a/build_analitycs/build_analytics/models/enums.py b/build_analitycs/build_analytics/models/enums.py deleted file mode 100644 index 5b18fdf..0000000 --- a/build_analitycs/build_analytics/models/enums.py +++ /dev/null @@ -1,17 +0,0 @@ -from enum import IntEnum - - -class ArchEnum(IntEnum): - i686 = 0 - x86_64 = 1 - aarch64 = 2 - ppc64le = 3 - s390x = 4 - - -class BuildTaskEnum(IntEnum): - idle = 0 - started = 1 - completed = 2 - failed = 3 - excluded = 4 diff --git a/build_analitycs/releases.txt b/build_analitycs/releases.txt deleted file mode 100644 index 416adfb..0000000 --- a/build_analitycs/releases.txt +++ /dev/null @@ -1,2 +0,0 @@ -0.1.0 (2023-03-01) -First version \ No newline at end of file diff --git a/build_analitycs/__init__.py b/build_analytics/__init__.py similarity index 100% rename from build_analitycs/__init__.py rename to build_analytics/__init__.py diff --git a/build_analitycs/build_analytics/__init__.py b/build_analytics/build_analytics/__init__.py similarity index 100% rename from build_analitycs/build_analytics/__init__.py rename to build_analytics/build_analytics/__init__.py diff --git a/build_analitycs/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py similarity index 54% rename from build_analitycs/build_analytics/api_client.py rename to build_analytics/build_analytics/api_client.py index bcc60ad..c72dca8 100644 --- a/build_analitycs/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -1,14 +1,15 @@ from datetime import datetime import logging - from urllib.parse import urljoin from typing import Dict, List +import requests from .models.build import Build from .models.build_task import BuildTask - -import requests +from .models.build_node_stats import BuildNodeStats +from .models.build_stat import BuildStat +from .models.web_node_stats import WebNodeStats TZ_OFFSET = '+00:00' @@ -51,23 +52,61 @@ class APIclient(): response.raise_for_status() return self._parse_build(response.json()) + def __parse_build_node_stats(self, stats: Dict) -> BuildNodeStats: + + keys = ['build_all', 'build_binaries', 'build_packages', 'build_srpm', 'build_node_task', + 'cas_notarize_artifacts', 'cas_source_authenticate', 'git_checkout', 'upload'] + params = {} + for k in keys: + try: + params[k] = BuildStat( + start_ts=datetime.fromisoformat( + stats[k]['start_ts']+TZ_OFFSET) if stats[k]['start_ts'] else None, + end_ts=datetime.fromisoformat( + stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) + except KeyError: + params[k] = BuildStat() + return BuildNodeStats(**params) + + def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats: + keys = ['build_done', 'logs_processing', 'packages_processing'] + params = {} + for k in keys: + try: + params[k] = BuildStat( + start_ts=datetime.fromisoformat( + stats[k]['start_ts']+TZ_OFFSET) if stats[k]['start_ts'] else None, + end_ts=datetime.fromisoformat( + stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) + except KeyError: + params[k] = BuildStat() + return WebNodeStats(**params) + def _parse_build_tasks(self, tasks_json: Dict, build_id: int) -> List[BuildTask]: result = [] for task in tasks_json: try: started_at = datetime.fromisoformat( - task['started_at']+TZ_OFFSET) \ - if task['started_at'] else None - finished_at = datetime.fromisoformat(task['finished_at']+TZ_OFFSET) \ - if task['finished_at'] else None + task['started_at']+TZ_OFFSET) if task['started_at'] else None + finished_at = datetime.fromisoformat( + task['finished_at']+TZ_OFFSET) if task['finished_at'] else None name = task['ref']['url'].split('/')[-1].replace('.git', '') + if not task['performance_stats']: + logging.warning( + "no perfomance_stats for build_id: %s, build_task_id: %s", build_id, task['id']) + stats = {'build_node_stats': {}, 'build_done_stats': {}} + else: + stats = task['performance_stats'][0]['statistics'] + params = {'id': task['id'], 'name': name, 'build_id': build_id, 'started_at': started_at, 'finished_at': finished_at, 'arch': task['arch'], - 'status_id': task['status']} + 'status_id': task['status'], + 'build_node_stats': self.__parse_build_node_stats(stats['build_node_stats']), + 'web_node_stats': self.__parse_web_node_stats(stats['build_done_stats'])} result.append(BuildTask(**params)) except Exception as err: # pylint: disable=broad-except logging.error("Cant convert build_task JSON %s (build_id %s) to BuildTask model: %s", @@ -79,8 +118,8 @@ class APIclient(): def _parse_build(self, build_json: Dict) -> Build: url = f"https://build.almalinux.org/build/{build_json['id']}" created_at = datetime.fromisoformat(build_json['created_at']+TZ_OFFSET) - finished_at = datetime.fromisoformat(build_json['finished_at']+TZ_OFFSET) \ - if build_json['finished_at'] else None + finished_at = datetime.fromisoformat( + build_json['finished_at']+TZ_OFFSET) if build_json['finished_at'] else None build_tasks = self._parse_build_tasks( build_json['tasks'], build_json['id']) diff --git a/build_analitycs/build_analytics/db.py b/build_analytics/build_analytics/db.py similarity index 62% rename from build_analitycs/build_analytics/db.py rename to build_analytics/build_analytics/db.py index 3c5e948..dac6d6a 100644 --- a/build_analitycs/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -1,11 +1,13 @@ from datetime import datetime -from typing import Union, Dict +from typing import Union, Dict, List import psycopg2 from .models.build_db import BuildDB from .models.build_task_db import BuildTaskDB +from .models.build_node_stat_db import BuildNodeStatDB from .models.db_config import DbConfig +from .models.web_node_stat_db import WebNodeStatDB class DB(): @@ -33,15 +35,37 @@ class DB(): build.created_at, build.finished_at)) self.__conn.commit() - def insert_buildtask(self, build_task: BuildTaskDB): + def insert_buildtask(self, build_task: BuildTaskDB, web_node_stats: List[WebNodeStatDB], + build_node_stats: List[BuildNodeStatDB]): + + cur = self.__conn.cursor() + # inserting build_task sql = ''' INSERT INTO build_tasks(id, name, build_id, arch_id, started_at, finished_at, status_id) VALUES (%s, %s, %s, %s, %s, %s, %s); ''' - - cur = self.__conn.cursor() cur.execute(sql, (build_task.id, build_task.name, build_task.build_id, build_task.arch_id, build_task.started_at, build_task.finished_at, build_task.status_id)) + + # inserting web node stats + for stat in web_node_stats: + sql = ''' + INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%s, %s, %s, %s); + ''' + cur.execute(sql, (stat.build_task_id, stat.stat_name_id, + stat.start_ts, stat.end_ts)) + + # inserting build node stats + for stat in build_node_stats: + sql = ''' + INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%s, %s, %s, %s); + ''' + cur.execute(sql, (stat.build_task_id, stat.stat_name_id, + stat.start_ts, stat.end_ts)) + + # commiting changes self.__conn.commit() def get_latest_build_id(self) -> Union[int, None]: @@ -101,7 +125,12 @@ class DB(): cur.execute(sql, (build.finished_at, build.id)) self.__conn.commit() - def update_build_task(self, build: BuildTaskDB): + def update_build_task(self, build_task: BuildTaskDB, + web_node_stats: List[WebNodeStatDB], + build_node_stats: List[BuildNodeStatDB]): + cur = self.__conn.cursor() + + # updating build_task sql = ''' UPDATE build_tasks SET status_id = %s, @@ -109,7 +138,28 @@ class DB(): finished_at = %s WHERE id = %s; ''' - cur = self.__conn.cursor() - cur.execute(sql, (build.status_id, build.started_at, - build.finished_at, build.id)) + cur.execute(sql, (build_task.status_id, build_task.started_at, + build_task.finished_at, build_task.id)) + + # updating web_node_stats + for stat in web_node_stats: + sql = ''' + UPDATE web_node_stats + SET start_ts = %s, + end_ts = %s + WHERE build_task_id = %s; + ''' + cur.execute(sql, (stat.start_ts, stat.end_ts)) + + # updating build_node_stats + for stat in build_node_stats: + sql = ''' + UPDATE build_node_stats + SET start_ts = %s, + end_ts = %s, + WHERE build_task_id = %s; + ''' + cur.execute(sql, (stat.start_ts, stat.end_ts)) + + # commiting changes self.__conn.commit() diff --git a/build_analitycs/build_analytics/extractor/__jnit__.py b/build_analytics/build_analytics/extractor/__jnit__.py similarity index 100% rename from build_analitycs/build_analytics/extractor/__jnit__.py rename to build_analytics/build_analytics/extractor/__jnit__.py diff --git a/build_analitycs/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py similarity index 59% rename from build_analitycs/build_analytics/extractor/extractor.py rename to build_analytics/build_analytics/extractor/extractor.py index 82b4b9b..f210692 100644 --- a/build_analitycs/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -12,6 +12,7 @@ from ..api_client import APIclient class Extractor: def __init__(self, config: ExtractorConfig, api: APIclient, db: DB): + self.start_from = config.start_from self.oldest_build_age = config.oldest_build_age self.api = api self.db = db @@ -21,7 +22,7 @@ class Extractor: page_num = 1 last_build_id = self.db.get_latest_build_id() if not last_build_id: - last_build_id = 0 + last_build_id = self.start_from logging.info("last_build_id: %s", last_build_id) stop = False @@ -34,11 +35,25 @@ class Extractor: stop = True break - # inserting build and build tasks + # inserting build build tasks and build tasks statistics logging.info("inserting %s", build.id) - self.db.insert_build(build.as_db_model()) + try: + self.db.insert_build(build.as_db_model()) + except Exception as error: # pylint: disable=broad-except + logging.error('failed to insert build %d: %s', + build.id, error, exc_info=True) + continue + for build_task in build.build_tasks: - self.db.insert_buildtask(build_task.as_db_model()) + try: + self.db.insert_buildtask(build_task.as_db_model(), + build_task.web_node_stats.as_db_model( + build_task.id), + build_task.web_node_stats.as_db_model( + build_task.id)) + except Exception as error: # pylint: disable=broad-except + logging.error('failed to insert build task %d: %s', + build_task.id, error, exc_info=True) build_count += 1 page_num += 1 return build_count @@ -49,25 +64,31 @@ class Extractor: removed_count = self.db.cleanup_builds(self.oldest_build_age) logging.info('removed %d entries', removed_count) - def __update_build_tasks_statuses(self, build_tasks: List[BuildTask], - build_tasks_status_db: Dict[int, int]): + def __update_build_tasks(self, build_tasks: List[BuildTask], + build_tasks_status_db: Dict[int, int]): for b in build_tasks: if b.status_id != build_tasks_status_db[b.id]: - logging.info('build taks %d status have changed %s -> %s. Updating DB', + logging.info('build: %s, build task %d status have changed %s -> %s. Updating DB', + b.build_id, b.id, BuildTaskEnum( build_tasks_status_db[b.id]).name, BuildTaskEnum(b.status_id).name) try: - self.db.update_build_task(b.as_db_model()) + self.db.update_build_task(b.as_db_model(), + b.web_node_stats.as_db_model( + b.id), + b.build_node_stats.as_db_model(b.id)) except Exception as err: # pylint: disable=broad-except logging.error( - 'failed to update build task %d: %s', - b.id, err, exc_info=True) + 'build: %d, failed to update build task %d: %s', + b.build_id, b.id, err, exc_info=True) else: - logging.info('build task %d was updated', b.id) + logging.info( + 'build: %d, build task %d was updated', b.build_id, b.id) else: logging.info( - "build_task %d is still %s. Skipping", b.id, BuildTaskEnum(b.status_id).name) + "build: %d, build_task %d is still %s. Skipping", + b.build_id, b.id, BuildTaskEnum(b.status_id).name) def update_builds(self): logging.info('Getting list of tasks from DB') @@ -80,7 +101,7 @@ class Extractor: logging.info('Updating build tasks') build_tasks_to_check = [ b for b in build.build_tasks if b.id in build_tasks_db] - self.__update_build_tasks_statuses( + self.__update_build_tasks( build_tasks_to_check, build_tasks_db) if build.finished_at: diff --git a/build_analitycs/build_analytics/extractor/start.py b/build_analytics/build_analytics/extractor/start.py similarity index 100% rename from build_analitycs/build_analytics/extractor/start.py rename to build_analytics/build_analytics/extractor/start.py diff --git a/build_analitycs/build_analytics/models/__init__.py b/build_analytics/build_analytics/models/__init__.py similarity index 100% rename from build_analitycs/build_analytics/models/__init__.py rename to build_analytics/build_analytics/models/__init__.py diff --git a/build_analitycs/build_analytics/models/build.py b/build_analytics/build_analytics/models/build.py similarity index 100% rename from build_analitycs/build_analytics/models/build.py rename to build_analytics/build_analytics/models/build.py diff --git a/build_analitycs/build_analytics/models/build_db.py b/build_analytics/build_analytics/models/build_db.py similarity index 100% rename from build_analitycs/build_analytics/models/build_db.py rename to build_analytics/build_analytics/models/build_db.py diff --git a/build_analytics/build_analytics/models/build_node_stat_db.py b/build_analytics/build_analytics/models/build_node_stat_db.py new file mode 100644 index 0000000..5e0fb5a --- /dev/null +++ b/build_analytics/build_analytics/models/build_node_stat_db.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel # pylint: disable=no-name-in-module +from typing import Optional + + +class BuildNodeStatDB(BaseModel): + """ + Build node stat as it sent to/received from database + """ + build_task_id: int + stat_name_id: int + start_ts: Optional[float] = None + end_ts: Optional[float] = None diff --git a/build_analytics/build_analytics/models/build_node_stats.py b/build_analytics/build_analytics/models/build_node_stats.py new file mode 100644 index 0000000..a71715d --- /dev/null +++ b/build_analytics/build_analytics/models/build_node_stats.py @@ -0,0 +1,41 @@ +from typing import List + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +from .build_stat import BuildStat +from .build_node_stat_db import BuildNodeStatDB +from .enums import BuildNodeStatsEnum + + +class BuildNodeStats(BaseModel): + """ + Represents build statistics for build node + """ + build_all: BuildStat + build_binaries: BuildStat + build_packages: BuildStat + build_srpm: BuildStat + build_node_task: BuildStat + cas_notarize_artifacts: BuildStat + cas_source_authenticate: BuildStat + git_checkout: BuildStat + upload: BuildStat + + def as_db_model(self, build_task_id: int) -> List[BuildNodeStatDB]: + result = [] + for field_name in self.__fields__.keys(): + + stats: BuildStat = getattr(self, field_name) + start_ts = stats.start_ts.timestamp() \ + if stats.start_ts else None + end_ts = stats.end_ts.timestamp() \ + if stats.end_ts else None + stat_name_id = BuildNodeStatsEnum[field_name].value + + build_node_stat_db = BuildNodeStatDB(build_task_id=build_task_id, + stat_name_id=stat_name_id, + start_ts=start_ts, + end_ts=end_ts) + result.append(build_node_stat_db) + return result diff --git a/build_analytics/build_analytics/models/build_stat.py b/build_analytics/build_analytics/models/build_stat.py new file mode 100644 index 0000000..ab1fa7f --- /dev/null +++ b/build_analytics/build_analytics/models/build_stat.py @@ -0,0 +1,15 @@ +""" +Module for BuildStat model +""" + +from datetime import datetime +from typing import Optional +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +class BuildStat(BaseModel): + """ + BuildStat represents particular build statistic + """ + start_ts: Optional[datetime] = None + end_ts: Optional[datetime] = None diff --git a/build_analytics/build_analytics/models/build_stat_db.py b/build_analytics/build_analytics/models/build_stat_db.py new file mode 100644 index 0000000..5b7240f --- /dev/null +++ b/build_analytics/build_analytics/models/build_stat_db.py @@ -0,0 +1,16 @@ +""" +Module for BuildStatDB model +""" + + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +class BuildStatDB(BaseModel): + """ + Represents build stat as it send to/received from database + """ + build_task_id: int + stat_name_id: int + start_ts: float + end_ts: float diff --git a/build_analitycs/build_analytics/models/build_task.py b/build_analytics/build_analytics/models/build_task.py similarity index 82% rename from build_analitycs/build_analytics/models/build_task.py rename to build_analytics/build_analytics/models/build_task.py index d4016d4..c84378e 100644 --- a/build_analitycs/build_analytics/models/build_task.py +++ b/build_analytics/build_analytics/models/build_task.py @@ -1,10 +1,12 @@ from datetime import datetime -from typing import Optional +from typing import Optional, Tuple from pydantic import BaseModel # pylint: disable=no-name-in-module from .build_task_db import BuildTaskDB +from .build_node_stats import BuildNodeStats from .enums import ArchEnum +from .web_node_stats import WebNodeStats class BuildTask(BaseModel): @@ -15,6 +17,8 @@ class BuildTask(BaseModel): started_at: Optional[datetime] = None finished_at: Optional[datetime] = None status_id: int + build_node_stats: BuildNodeStats + web_node_stats: WebNodeStats def as_db_model(self) -> BuildTaskDB: started_at = self.started_at.timestamp() \ diff --git a/build_analitycs/build_analytics/models/build_task_db.py b/build_analytics/build_analytics/models/build_task_db.py similarity index 100% rename from build_analitycs/build_analytics/models/build_task_db.py rename to build_analytics/build_analytics/models/build_task_db.py diff --git a/build_analitycs/build_analytics/models/db_config.py b/build_analytics/build_analytics/models/db_config.py similarity index 100% rename from build_analitycs/build_analytics/models/db_config.py rename to build_analytics/build_analytics/models/db_config.py diff --git a/build_analytics/build_analytics/models/enums.py b/build_analytics/build_analytics/models/enums.py new file mode 100644 index 0000000..0b63a09 --- /dev/null +++ b/build_analytics/build_analytics/models/enums.py @@ -0,0 +1,37 @@ +# pylint: disable=invalid-name + +from enum import IntEnum + + +class ArchEnum(IntEnum): + i686 = 0 + x86_64 = 1 + aarch64 = 2 + ppc64le = 3 + s390x = 4 + + +class BuildTaskEnum(IntEnum): + idle = 0 + started = 1 + completed = 2 + failed = 3 + excluded = 4 + + +class WebNodeStatsEnum(IntEnum): + build_done = 0 + logs_processing = 1 + packages_processing = 2 + + +class BuildNodeStatsEnum(IntEnum): + upload = 0 + build_all = 1 + build_srpm = 2 + git_checkout = 3 + build_binaries = 4 + build_packages = 5 + build_node_task = 6 + cas_notarize_artifacts = 7 + cas_source_authenticate = 8 diff --git a/build_analitycs/build_analytics/models/extractor_config.py b/build_analytics/build_analytics/models/extractor_config.py similarity index 82% rename from build_analitycs/build_analytics/models/extractor_config.py rename to build_analytics/build_analytics/models/extractor_config.py index 8b8f58a..a49c3dd 100644 --- a/build_analitycs/build_analytics/models/extractor_config.py +++ b/build_analytics/build_analytics/models/extractor_config.py @@ -10,6 +10,7 @@ ALBS_URL_DEFAULT = 'https://build.almalinux.org' LOG_FILE_DEFAULT = '/tmp/extractor.log' API_DEFAULT = 30 SCRAPE_INTERVAL_DEFAULT = 3600 +START_FROM_DEFAULT = 5808 class ExtractorConfig(BaseModel): @@ -21,7 +22,7 @@ class ExtractorConfig(BaseModel): albs_url: HttpUrl = Field(description='ALBS root URL', default=ALBS_URL_DEFAULT) oldest_build_age: datetime = \ - Field(description='oldest build age to extract and store') + Field(description='oldest build age to store') jwt: str = Field(description='ALBS JWT token') db_config: DbConfig = Field(description="database configuration") api_timeout: int = Field( @@ -29,3 +30,5 @@ class ExtractorConfig(BaseModel): default=API_DEFAULT) scrape_interval: int = Field(description='how often (in seconds) we will extract data from ALBS', default=SCRAPE_INTERVAL_DEFAULT) + start_from: int = Field(description='build id to start populating empty db with', + default=START_FROM_DEFAULT) diff --git a/build_analitycs/build_analytics/models/sign_task_db.py b/build_analytics/build_analytics/models/sign_task_db.py similarity index 100% rename from build_analitycs/build_analytics/models/sign_task_db.py rename to build_analytics/build_analytics/models/sign_task_db.py diff --git a/build_analytics/build_analytics/models/web_node_stat_db.py b/build_analytics/build_analytics/models/web_node_stat_db.py new file mode 100644 index 0000000..dd28225 --- /dev/null +++ b/build_analytics/build_analytics/models/web_node_stat_db.py @@ -0,0 +1,13 @@ + +from pydantic import BaseModel # pylint: disable=no-name-in-module +from typing import Optional + + +class WebNodeStatDB(BaseModel): + """ + Represents WebNodeStat as it sent to/received from databse + """ + build_task_id: int + stat_name_id: int + start_ts: Optional[float] = None + end_ts: Optional[float] = None diff --git a/build_analytics/build_analytics/models/web_node_stats.py b/build_analytics/build_analytics/models/web_node_stats.py new file mode 100644 index 0000000..46f77c6 --- /dev/null +++ b/build_analytics/build_analytics/models/web_node_stats.py @@ -0,0 +1,35 @@ +from typing import List + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +from .build_stat import BuildStat +from .web_node_stat_db import WebNodeStatDB +from .enums import WebNodeStatsEnum + + +class WebNodeStats(BaseModel): + """ + Represents build statistics for web node + """ + build_done: BuildStat + logs_processing: BuildStat + packages_processing: BuildStat + + def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]: + result = [] + for field_name in self.__fields__.keys(): + + stats: BuildStat = getattr(self, field_name) + start_ts = stats.start_ts.timestamp() \ + if stats.start_ts else None + end_ts = stats.end_ts.timestamp() \ + if stats.end_ts else None + stat_name_id = WebNodeStatsEnum[field_name].value + + web_node_stat_db = WebNodeStatDB(build_task_id=build_task_id, + stat_name_id=stat_name_id, + start_ts=start_ts, + end_ts=end_ts) + result.append(web_node_stat_db) + return result diff --git a/build_analitycs/config_default.yml b/build_analytics/config_default.yml similarity index 87% rename from build_analitycs/config_default.yml rename to build_analytics/config_default.yml index 70349bc..c7b5e66 100644 --- a/build_analitycs/config_default.yml +++ b/build_analytics/config_default.yml @@ -55,4 +55,9 @@ data_store_days: 30 # sleep time in seconds between data extraction # required: no # default: 3600 -scrape_interval: 3600 \ No newline at end of file +scrape_interval: 3600 + +# build_id to start populating empty db with +# required: false +# default: 5808 (first build with correct metrics) +start_from: 5808 diff --git a/build_analitycs/db_schema/postgres.sql b/build_analytics/migrations/1.sql similarity index 51% rename from build_analitycs/db_schema/postgres.sql rename to build_analytics/migrations/1.sql index 036e972..40c6029 100644 --- a/build_analitycs/db_schema/postgres.sql +++ b/build_analytics/migrations/1.sql @@ -1,5 +1,6 @@ +BEGIN; + -- builds -DROP TABLE IF EXISTS builds CASCADE; CREATE TABLE builds ( id INTEGER PRIMARY KEY, url VARCHAR(50) NOT NULL, @@ -16,7 +17,6 @@ ON builds(finished_at); -- build_taks_enum -DROP TABLE IF EXISTS build_task_status_enum CASCADE; CREATE TABLE IF NOT EXISTS build_task_status_enum( id INTEGER PRIMARY KEY, value VARCHAR(15) @@ -32,7 +32,6 @@ VALUES -- arch_enum -DROP TABLE IF EXISTS arch_enum CASCADE; CREATE TABLE arch_enum( id INTEGER PRIMARY KEY, value VARCHAR(15) @@ -47,8 +46,39 @@ VALUES (4, 's390x'); +-- web_node_stats_enum +CREATE TABLE web_node_stats_enum ( + id INTEGER PRIMARY KEY, + value VARCHAR(50) +); + +INSERT INTO web_node_stats_enum (id, value) +VALUEs + (0, 'build_done'), + (1, 'logs_processing'), + (2, 'packages_processing'); + + +-- build_node_stats_enum +CREATE TABLE build_node_stats_enum( + id INTEGER PRIMARY KEY, + value VARCHAR(50) +); + +INSERT INTO build_node_stats_enum (id, value) +VALUES + (0, 'upload'), + (1, 'build_all'), + (2, 'build_srpm'), + (3, 'git_checkout'), + (4, 'build_binaries'), + (5, 'build_packages'), + (6, 'build_node_task'), + (7, 'cas_notarize_artifacts'), + (8, 'cas_source_authenticate'); + + -- build_tasks -DROP TABLE IF EXISTS build_tasks CASCADE; CREATE TABLE build_tasks ( id INTEGER PRIMARY KEY, name VARCHAR(50) NOT NULL, @@ -69,8 +99,43 @@ CREATE INDEX build_tasks_finished_at ON build_tasks(finished_at); +-- web_node_stats +CREATE TABLE web_node_stats ( + build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, + stat_name_id INTEGER REFERENCES web_node_stats_enum(id) ON DELETE SET NULL, + start_ts REAL, + end_ts REAL +); + +CREATE INDEX web_node_stats_build_task_id +ON web_node_stats(build_task_id); + +CREATE INDEX web_node_stats_start_ts +ON web_node_stats(start_ts); + +CREATE INDEX web_node_stats_end_ts +ON web_node_stats(end_ts); + + +-- build_node_stats +CREATE TABLE build_node_stats ( + build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, + stat_name_id INTEGER REFERENCES build_node_stats_enum(id) ON DELETE SET NULL, + start_ts REAL, + end_ts REAL +); + +CREATE INDEX build_node_stats_build_task_id +ON build_node_stats(build_task_id); + +CREATE INDEX build_node_stats_build_start_ts +ON build_node_stats(start_ts); + +CREATE INDEX build_node_stats_build_end_ts +ON build_node_stats(end_ts); + + -- sign_tasks -DROP TABLE IF EXISTS sign_tasks CASCADE; CREATE TABLE sign_tasks ( id INTEGER PRIMARY KEY, build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE, @@ -90,3 +155,14 @@ ON sign_tasks(started_at); CREATE INDEX sign_tasks_finished_at ON sign_tasks(finished_at); + + + +-- schema_version +CREATE TABLE schema_version ( + version INTEGER +); +INSERT INTO schema_version (version) +VALUES (1); + +COMMIT; \ No newline at end of file diff --git a/build_analytics/releases.txt b/build_analytics/releases.txt new file mode 100644 index 0000000..409ef5f --- /dev/null +++ b/build_analytics/releases.txt @@ -0,0 +1,5 @@ +0.1.0 (2023-03-01) +First version + +0.2.0 +New parameter start_from \ No newline at end of file diff --git a/build_analitycs/requirements.txt b/build_analytics/requirements.txt similarity index 100% rename from build_analitycs/requirements.txt rename to build_analytics/requirements.txt diff --git a/build_analitycs/run_extractor.py b/build_analytics/run_extractor.py similarity index 100% rename from build_analitycs/run_extractor.py rename to build_analytics/run_extractor.py diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json new file mode 100644 index 0000000..c0748d8 --- /dev/null +++ b/grafana-dashbords/Build analytics.json @@ -0,0 +1,1509 @@ +{ + "__inputs": [ + { + "name": "DS_ALBS_ANALYTICS", + "label": "albs_analytics", + "description": "", + "type": "datasource", + "pluginId": "postgres", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.6" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "postgres", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "finished" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS finished\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at is NOT NULL;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS running\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at is NULL;", + "refId": "C", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Builds", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS idle FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 0;", + "refId": "idle", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS started FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 1;", + "refId": "started", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS completed FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 2;", + "refId": "completed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS failed FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 3;", + "refId": "failed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS excluded FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 4;", + "refId": "excluded", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by status", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "x86_64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "i686" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "aarch64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ppc64le" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS i686 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 0;", + "refId": "i686", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS x86_64 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 1;", + "refId": "x86_64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS aarch64 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 2;", + "refId": "aarch64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS ppc64le FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 3;", + "refId": "ppc64le", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS s390x FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 4;", + "refId": "excluded", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by arch", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MAX(finished_at - created_at) AS MAX\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "MAX", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MIN(finished_at - created_at) AS MIN\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "MIN", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT AVG(finished_at - created_at) AS AVG\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "AVG", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build duration", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "MAX", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "MIN", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT AVG(finished_at - started_at) AS AVG\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "AVG", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build task duration", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 5, + "panels": [], + "title": "Build details", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished builds", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "created" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 7, + "panels": [], + "title": "Build tasks details", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 15, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE $__unixEpochFilter(started_at) AND status_id >= 2;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished buld tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Build analytics", + "uid": "02mg4oxVk", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana-dashbords/albs_analytics.json b/grafana-dashbords/albs_analytics.json deleted file mode 100644 index 6aa83f8..0000000 --- a/grafana-dashbords/albs_analytics.json +++ /dev/null @@ -1,280 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_POSTGRESQL", - "label": "PostgreSQL", - "description": "", - "type": "datasource", - "pluginId": "postgres", - "pluginName": "PostgreSQL" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "9.3.2" - }, - { - "type": "datasource", - "id": "postgres", - "name": "PostgreSQL", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "custom.width", - "value": 54 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "created_at" - }, - "properties": [ - { - "id": "custom.width", - "value": 226 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "finished_at" - }, - "properties": [ - { - "id": "custom.width", - "value": 209 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "finished" - }, - "properties": [ - { - "id": "custom.width", - "value": 187 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "created" - }, - "properties": [ - { - "id": "custom.width", - "value": 213 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "url" - }, - "properties": [ - { - "id": "custom.width", - "value": 279 - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration (h)" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "hide": false, - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT id, url, created_at * 1000 as created, finished_at * 1000 as finished, (finished_at - created_at) / (60*60) as duration\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Finished builds", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "created" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "duration": "duration (h)" - } - } - } - ], - "type": "table" - } - ], - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "albs_analytics", - "uid": "02mg4oxVk", - "version": 1, - "weekStart": "" -} \ No newline at end of file -- 2.43.5 From c6a14cfe3ac59e086abcbbdf6dacc3540b71928f Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Tue, 7 Mar 2023 16:02:14 +0100 Subject: [PATCH 02/12] - fixed bug with wrong ts of build_tasks steps - db: moved to double percicion for all ts columns - added db_schema_version check --- build_analytics/build_analytics/api_client.py | 10 +++++-- .../{models/enums.py => const.py} | 4 +++ build_analytics/build_analytics/db.py | 26 ++++++++++++++++--- .../build_analytics/extractor/extractor.py | 6 ++--- .../build_analytics/extractor/start.py | 18 +++++++++++-- .../models/build_node_stats.py | 2 +- .../build_analytics/models/build_task.py | 4 +-- .../build_analytics/models/web_node_stats.py | 2 +- build_analytics/config_default.yml | 2 +- build_analytics/migrations/1.sql | 21 +++++++-------- build_analytics/releases.txt | 3 ++- 11 files changed, 70 insertions(+), 28 deletions(-) rename build_analytics/build_analytics/{models/enums.py => const.py} (91%) diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index c72dca8..f30adbd 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -53,6 +53,7 @@ class APIclient(): return self._parse_build(response.json()) def __parse_build_node_stats(self, stats: Dict) -> BuildNodeStats: + logging.debug('raw json: %s', stats) keys = ['build_all', 'build_binaries', 'build_packages', 'build_srpm', 'build_node_task', 'cas_notarize_artifacts', 'cas_source_authenticate', 'git_checkout', 'upload'] @@ -66,11 +67,14 @@ class APIclient(): stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) except KeyError: params[k] = BuildStat() - return BuildNodeStats(**params) + build_node_stats = BuildNodeStats(**params) + logging.debug('BuildNodeStats: %s', build_node_stats) + return build_node_stats def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats: keys = ['build_done', 'logs_processing', 'packages_processing'] params = {} + logging.debug('raw json: %s', stats) for k in keys: try: params[k] = BuildStat( @@ -80,7 +84,9 @@ class APIclient(): stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) except KeyError: params[k] = BuildStat() - return WebNodeStats(**params) + web_node_stats = WebNodeStats(**params) + logging.debug('WebNodeStats %s', web_node_stats) + return web_node_stats def _parse_build_tasks(self, tasks_json: Dict, build_id: int) -> List[BuildTask]: result = [] diff --git a/build_analytics/build_analytics/models/enums.py b/build_analytics/build_analytics/const.py similarity index 91% rename from build_analytics/build_analytics/models/enums.py rename to build_analytics/build_analytics/const.py index 0b63a09..c29d1d9 100644 --- a/build_analytics/build_analytics/models/enums.py +++ b/build_analytics/build_analytics/const.py @@ -2,7 +2,11 @@ from enum import IntEnum +# supported schema version +DB_SCHEMA_VER = 1 + +# ENUMS class ArchEnum(IntEnum): i686 = 0 x86_64 = 1 diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index dac6d6a..f2a12a1 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -1,5 +1,6 @@ from datetime import datetime -from typing import Union, Dict, List +from typing import Union, Dict, List, Optional +import logging import psycopg2 @@ -55,15 +56,18 @@ class DB(): ''' cur.execute(sql, (stat.build_task_id, stat.stat_name_id, stat.start_ts, stat.end_ts)) + logging.debug('raw SQL query: %s', cur.query) # inserting build node stats for stat in build_node_stats: + logging.debug('BuildNodeStats: %s', stat) sql = ''' INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) VALUES (%s, %s, %s, %s); ''' cur.execute(sql, (stat.build_task_id, stat.stat_name_id, stat.start_ts, stat.end_ts)) + logging.debug('raw SQL query: %s', cur.query) # commiting changes self.__conn.commit() @@ -149,17 +153,31 @@ class DB(): end_ts = %s WHERE build_task_id = %s; ''' - cur.execute(sql, (stat.start_ts, stat.end_ts)) + cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) # updating build_node_stats for stat in build_node_stats: sql = ''' UPDATE build_node_stats SET start_ts = %s, - end_ts = %s, + end_ts = %s WHERE build_task_id = %s; ''' - cur.execute(sql, (stat.start_ts, stat.end_ts)) + cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) # commiting changes self.__conn.commit() + + def get_db_schema_version(self) -> Optional[int]: + sql = ''' + SELECT * + FROM schema_version + LIMIT 1; + ''' + cur = self.__conn.cursor() + cur.execute(sql) + val = cur.fetchone() + print(val) + if not val: + return None + return int(val[0]) diff --git a/build_analytics/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py index f210692..3a3a953 100644 --- a/build_analytics/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -4,7 +4,7 @@ import logging from typing import List, Dict from ..models.extractor_config import ExtractorConfig -from ..models.enums import BuildTaskEnum +from ..const import BuildTaskEnum from ..models.build import BuildTask from ..db import DB from ..api_client import APIclient @@ -22,7 +22,7 @@ class Extractor: page_num = 1 last_build_id = self.db.get_latest_build_id() if not last_build_id: - last_build_id = self.start_from + last_build_id = self.start_from - 1 logging.info("last_build_id: %s", last_build_id) stop = False @@ -49,7 +49,7 @@ class Extractor: self.db.insert_buildtask(build_task.as_db_model(), build_task.web_node_stats.as_db_model( build_task.id), - build_task.web_node_stats.as_db_model( + build_task.build_node_stats.as_db_model( build_task.id)) except Exception as error: # pylint: disable=broad-except logging.error('failed to insert build task %d: %s', diff --git a/build_analytics/build_analytics/extractor/start.py b/build_analytics/build_analytics/extractor/start.py index 5467a1c..90f59c1 100644 --- a/build_analytics/build_analytics/extractor/start.py +++ b/build_analytics/build_analytics/extractor/start.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta import logging from logging.handlers import RotatingFileHandler +import sys import time import yaml @@ -8,6 +9,7 @@ import yaml # pylint: disable=relative-beyond-top-level from ..api_client import APIclient from ..db import DB +from ..const import DB_SCHEMA_VER from .extractor import Extractor from ..models.extractor_config import ExtractorConfig from ..models.db_config import DbConfig @@ -46,16 +48,28 @@ def start(yml_path: str): # configuring logging logging.basicConfig(level=logging.INFO, - format='%(asctime)s %(levelname)s <%(funcName)s> %(message)s', + format='%(asctime)s %(levelname)s %(funcName)s() %(message)s', handlers=[RotatingFileHandler(config.log_file, maxBytes=10000000, backupCount=3)]) + # some pre-flight checks + db = DB(config.db_config) + cur_version = db.get_db_schema_version() + if not cur_version: + logging.error( + 'Cant get db schema version. Make sure that schema_version exists') + sys.exit(1) + if cur_version != DB_SCHEMA_VER: + logging.error('unsupported DB schema: want %s, have %s', + DB_SCHEMA_VER, cur_version) + sys.exit(1) + while True: logging.info('Starting extraction proccess') api = APIclient(api_root=config.albs_url, jwt=config.jwt, timeout=config.api_timeout) - db = DB(config.db_config) + extractor = Extractor(config, api, db) logging.info('Starting builds insertion') diff --git a/build_analytics/build_analytics/models/build_node_stats.py b/build_analytics/build_analytics/models/build_node_stats.py index a71715d..39eb74e 100644 --- a/build_analytics/build_analytics/models/build_node_stats.py +++ b/build_analytics/build_analytics/models/build_node_stats.py @@ -5,7 +5,7 @@ from pydantic import BaseModel # pylint: disable=no-name-in-module from .build_stat import BuildStat from .build_node_stat_db import BuildNodeStatDB -from .enums import BuildNodeStatsEnum +from ..const import BuildNodeStatsEnum class BuildNodeStats(BaseModel): diff --git a/build_analytics/build_analytics/models/build_task.py b/build_analytics/build_analytics/models/build_task.py index c84378e..613c906 100644 --- a/build_analytics/build_analytics/models/build_task.py +++ b/build_analytics/build_analytics/models/build_task.py @@ -1,11 +1,11 @@ from datetime import datetime -from typing import Optional, Tuple +from typing import Optional from pydantic import BaseModel # pylint: disable=no-name-in-module from .build_task_db import BuildTaskDB from .build_node_stats import BuildNodeStats -from .enums import ArchEnum +from ..const import ArchEnum from .web_node_stats import WebNodeStats diff --git a/build_analytics/build_analytics/models/web_node_stats.py b/build_analytics/build_analytics/models/web_node_stats.py index 46f77c6..0476863 100644 --- a/build_analytics/build_analytics/models/web_node_stats.py +++ b/build_analytics/build_analytics/models/web_node_stats.py @@ -5,7 +5,7 @@ from pydantic import BaseModel # pylint: disable=no-name-in-module from .build_stat import BuildStat from .web_node_stat_db import WebNodeStatDB -from .enums import WebNodeStatsEnum +from ..const import WebNodeStatsEnum class WebNodeStats(BaseModel): diff --git a/build_analytics/config_default.yml b/build_analytics/config_default.yml index c7b5e66..8e68a61 100644 --- a/build_analytics/config_default.yml +++ b/build_analytics/config_default.yml @@ -60,4 +60,4 @@ scrape_interval: 3600 # build_id to start populating empty db with # required: false # default: 5808 (first build with correct metrics) -start_from: 5808 +start_from: diff --git a/build_analytics/migrations/1.sql b/build_analytics/migrations/1.sql index 40c6029..cf17985 100644 --- a/build_analytics/migrations/1.sql +++ b/build_analytics/migrations/1.sql @@ -4,8 +4,8 @@ BEGIN; CREATE TABLE builds ( id INTEGER PRIMARY KEY, url VARCHAR(50) NOT NULL, - created_at REAL NOT NULL, - finished_at REAL + created_at DOUBLE PRECISION NOT NULL, + finished_at DOUBLE PRECISION ); @@ -85,8 +85,8 @@ CREATE TABLE build_tasks ( build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE, arch_id INTEGER REFERENCES arch_enum(id) ON DELETE SET NULL, status_id INTEGER REFERENCES build_task_status_enum(id) ON DELETE SET NULL, - started_at REAL, - finished_at REAL + started_at DOUBLE PRECISION, + finished_at DOUBLE PRECISION ); CREATE INDEX build_tasks_build_id @@ -103,8 +103,8 @@ ON build_tasks(finished_at); CREATE TABLE web_node_stats ( build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, stat_name_id INTEGER REFERENCES web_node_stats_enum(id) ON DELETE SET NULL, - start_ts REAL, - end_ts REAL + start_ts DOUBLE PRECISION, + end_ts DOUBLE PRECISION ); CREATE INDEX web_node_stats_build_task_id @@ -121,8 +121,8 @@ ON web_node_stats(end_ts); CREATE TABLE build_node_stats ( build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, stat_name_id INTEGER REFERENCES build_node_stats_enum(id) ON DELETE SET NULL, - start_ts REAL, - end_ts REAL + start_ts DOUBLE PRECISION, + end_ts DOUBLE PRECISION ); CREATE INDEX build_node_stats_build_task_id @@ -140,8 +140,8 @@ CREATE TABLE sign_tasks ( id INTEGER PRIMARY KEY, build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE, buildtask_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, - started_at REAL, - finished_at REAL + started_at DOUBLE PRECISION, + finished_at DOUBLE PRECISION ); CREATE INDEX sign_tasks_build_id @@ -157,7 +157,6 @@ CREATE INDEX sign_tasks_finished_at ON sign_tasks(finished_at); - -- schema_version CREATE TABLE schema_version ( version INTEGER diff --git a/build_analytics/releases.txt b/build_analytics/releases.txt index 409ef5f..bb25103 100644 --- a/build_analytics/releases.txt +++ b/build_analytics/releases.txt @@ -2,4 +2,5 @@ First version 0.2.0 -New parameter start_from \ No newline at end of file +New parameter start_from +moved to double persition for timestamps \ No newline at end of file -- 2.43.5 From c4983d6ae49c093c494b20c213126b79c49d59fd Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Tue, 7 Mar 2023 17:48:19 +0100 Subject: [PATCH 03/12] ALBS-1026: new Dashboard Build task details --- grafana-dashbords/Build task details.json | 1019 +++++++++++++++++++++ 1 file changed, 1019 insertions(+) create mode 100644 grafana-dashbords/Build task details.json diff --git a/grafana-dashbords/Build task details.json b/grafana-dashbords/Build task details.json new file mode 100644 index 0000000..0223564 --- /dev/null +++ b/grafana-dashbords/Build task details.json @@ -0,0 +1,1019 @@ +{ + "__inputs": [ + { + "name": "DS_POSTGRESQL", + "label": "PostgreSQL", + "description": "", + "type": "datasource", + "pluginId": "postgres", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.2" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "postgres", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.id = $build_task;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Task summary", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 10, + "panels": [], + "title": "Tasks duration", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 0, + "y": 4 + }, + "id": 12, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_all\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 1", + "refId": "build_all", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_binaries\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 4", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_packages\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 5", + "refId": "build_packages", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_node_task\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 6", + "refId": "build_node_task", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as notarize_artifacts\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 7;", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build node", + "type": "bargauge" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "packages_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "logs_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 14, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as logs_processing\nFROM web_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 1", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as packages_processing\nFROM web_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2", + "refId": "packages_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Web node", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 8, + "panels": [], + "title": "Details", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", + "refId": "web node stats", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Web node stats", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "started" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", + "refId": "web node stats", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build node stats", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "definition": "SELECT id \nFROM builds\nORDER BY id DESC;", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "build", + "options": [], + "query": "SELECT id \nFROM builds\nORDER BY id DESC;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "definition": "SELECT id from build_tasks where build_id = $build", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "build_task", + "options": [], + "query": "SELECT id from build_tasks where build_id = $build", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Build task details", + "uid": "vtQClqxVk", + "version": 19, + "weekStart": "" +} \ No newline at end of file -- 2.43.5 From 76be50ca9cd5b36221866a842c0ac3d5d36c924e Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 8 Mar 2023 00:02:25 +0100 Subject: [PATCH 04/12] Grafana: Added new dashboards, improved old ones --- grafana-dashbords/Build analytics.json | 556 ++++++++++++-------- grafana-dashbords/Build details.json | 244 +++++++++ grafana-dashbords/Build task details.json | 584 +++++++++++++--------- 3 files changed, 951 insertions(+), 433 deletions(-) create mode 100644 grafana-dashbords/Build details.json diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json index c0748d8..7de3172 100644 --- a/grafana-dashbords/Build analytics.json +++ b/grafana-dashbords/Build analytics.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_ALBS_ANALYTICS", - "label": "albs_analytics", + "name": "DS_POSTGRESQL", + "label": "PostgreSQL", "description": "", "type": "datasource", "pluginId": "postgres", @@ -11,11 +11,17 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.3.6" + "version": "9.3.2" }, { "type": "panel", @@ -64,7 +70,7 @@ } ] }, - "editable": false, + "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -87,7 +93,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "fieldConfig": { "defaults": { @@ -171,7 +177,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -200,7 +206,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -233,7 +239,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -363,7 +369,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -392,7 +398,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -421,7 +427,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -450,7 +456,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -479,7 +485,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -512,7 +518,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -702,7 +708,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -731,7 +737,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -760,7 +766,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -789,7 +795,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -818,7 +824,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -851,7 +857,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "fieldConfig": { "defaults": { @@ -894,13 +900,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -913,7 +919,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MAX(finished_at - created_at) AS MAX\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "MAX", + "refId": "max", "sql": { "columns": [ { @@ -937,7 +943,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -951,7 +957,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MIN(finished_at - created_at) AS MIN\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "MIN", + "refId": "min", "sql": { "columns": [ { @@ -972,24 +978,16 @@ "urlPath": "" }, { - "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], "format": "table", "hide": false, - "method": "GET", - "queryParams": "", "rawQuery": true, - "rawSql": "SELECT AVG(finished_at - created_at) AS AVG\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "AVG", + "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", + "refId": "meidan", "sql": { "columns": [ { @@ -1006,8 +1004,7 @@ } ], "limit": 50 - }, - "urlPath": "" + } } ], "title": "Build duration", @@ -1016,7 +1013,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -1060,13 +1057,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1079,7 +1076,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", - "refId": "MAX", + "refId": "max", "sql": { "columns": [ { @@ -1103,7 +1100,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1117,7 +1114,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", - "refId": "MIN", + "refId": "min", "sql": { "columns": [ { @@ -1141,7 +1138,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1154,8 +1151,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT AVG(finished_at - started_at) AS AVG\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", - "refId": "AVG", + "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4) as durations\n\n\n", + "refId": "median", "sql": { "columns": [ { @@ -1180,22 +1177,308 @@ "type": "stat" }, { - "collapsed": false, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 8, "w": 24, "x": 0, "y": 15 }, - "id": 5, - "panels": [], - "title": "Build details", - "type": "row" + "id": 20, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build task steps duration (SUM)", + "type": "bargauge" }, { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -1260,6 +1543,23 @@ "value": 107 } ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" + } + ] + } + ] } ] }, @@ -1267,7 +1567,7 @@ "h": 15, "w": 24, "x": 0, - "y": 16 + "y": 23 }, "id": 2, "options": { @@ -1286,13 +1586,13 @@ } ] }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1347,146 +1647,6 @@ } ], "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 7, - "panels": [], - "title": "Build tasks details", - "type": "row" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 15, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "duration" - } - ] - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE $__unixEpochFilter(started_at) AND status_id >= 2;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Finished buld tasks", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - } - ], - "type": "table" } ], "refresh": "1m", @@ -1504,6 +1664,6 @@ "timezone": "", "title": "Build analytics", "uid": "02mg4oxVk", - "version": 3, + "version": 21, "weekStart": "" } \ No newline at end of file diff --git a/grafana-dashbords/Build details.json b/grafana-dashbords/Build details.json new file mode 100644 index 0000000..d39e49a --- /dev/null +++ b/grafana-dashbords/Build details.json @@ -0,0 +1,244 @@ +{ + "__inputs": [ + { + "name": "DS_POSTGRESQL", + "label": "PostgreSQL", + "description": "", + "type": "datasource", + "pluginId": "postgres", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.2" + }, + { + "type": "datasource", + "id": "postgres", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "build_id" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id AND status_id >= 2;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished buld tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "definition": "SELECT id\nFROM builds\nORDER BY id DESC", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "build_id", + "options": [], + "query": "SELECT id\nFROM builds\nORDER BY id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Build details", + "uid": "dmVtrz-4k", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana-dashbords/Build task details.json b/grafana-dashbords/Build task details.json index 0223564..3edd794 100644 --- a/grafana-dashbords/Build task details.json +++ b/grafana-dashbords/Build task details.json @@ -7,10 +7,22 @@ "type": "datasource", "pluginId": "postgres", "pluginName": "PostgreSQL" + }, + { + "name": "DS_EXPRESSION", + "label": "Expression", + "description": "", + "type": "datasource", + "pluginId": "__expr__" } ], "__elements": {}, "__requires": [ + { + "type": "datasource", + "id": "__expr__", + "version": "1.0.0" + }, { "type": "panel", "id": "bargauge", @@ -23,18 +35,18 @@ "name": "Grafana", "version": "9.3.2" }, - { - "type": "panel", - "id": "piechart", - "name": "Pie chart", - "version": "" - }, { "type": "datasource", "id": "postgres", "name": "PostgreSQL", "version": "1.0.0" }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, { "type": "panel", "id": "table", @@ -64,7 +76,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -194,19 +206,6 @@ ], "type": "table" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 10, - "panels": [], - "title": "Tasks duration", - "type": "row" - }, { "datasource": { "type": "postgres", @@ -233,10 +232,10 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 16, + "h": 9, + "w": 24, "x": 0, - "y": 4 + "y": 3 }, "id": 12, "options": { @@ -298,35 +297,6 @@ "table": "build_node_stats", "urlPath": "" }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": true, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as build_all\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 1", - "refId": "build_all", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, { "datasource": { "type": "postgres", @@ -414,64 +384,6 @@ "limit": 50 } }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": true, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as build_packages\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 5", - "refId": "build_packages", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": true, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as build_node_task\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 6", - "refId": "build_node_task", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, { "datasource": { "type": "postgres", @@ -529,111 +441,15 @@ ], "limit": 50 } - } - ], - "title": "Build node", - "type": "bargauge" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "packages_processing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "logs_processing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.2", - "targets": [ { - "cacheDurationSeconds": 300, "datasource": { "type": "postgres", "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], "format": "table", - "method": "GET", - "queryParams": "", + "hide": false, "rawQuery": true, "rawSql": "SELECT end_ts - start_ts as logs_processing\nFROM web_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 1", "refId": "logs_processing", @@ -653,8 +469,7 @@ } ], "limit": 50 - }, - "urlPath": "" + } }, { "datasource": { @@ -686,8 +501,8 @@ } } ], - "title": "Web node", - "type": "piechart" + "title": "Task duration", + "type": "bargauge" }, { "collapsed": false, @@ -746,6 +561,30 @@ "value": "s" } ] + }, + { + "matcher": { + "id": "byName", + "options": "finished" + }, + "properties": [ + { + "id": "custom.width", + "value": 235 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "started" + }, + "properties": [ + { + "id": "custom.width", + "value": 205 + } + ] } ] }, @@ -765,7 +604,8 @@ "show": false }, "frameIndex": 1, - "showHeader": true + "showHeader": true, + "sortBy": [] }, "pluginVersion": "9.3.2", "targets": [ @@ -872,6 +712,18 @@ "value": "s" } ] + }, + { + "matcher": { + "id": "byName", + "options": "started" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] } ] }, @@ -895,7 +747,7 @@ "sortBy": [ { "desc": true, - "displayName": "started" + "displayName": "duration" } ] }, @@ -959,6 +811,287 @@ } ], "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 6, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "finished" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", + "refId": "web node stats", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build node stats", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 3, + "y": 40 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n bt.finished_at - bt.started_at AS task_duration \nFROM build_tasks AS bt\nWHERE bt.id = $build_task;", + "refId": "task_duration", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n stats.end_ts - stats.start_ts AS web_node_duration\nFROM web_node_stats AS stats\nWHERE build_task_id = $build_task and stat_name_id=0;", + "refId": "web_node_duration", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n stats.end_ts - stats.start_ts AS build_node_duration\nFROM build_node_stats AS stats\nWHERE build_task_id = $build_task AND stat_name_id=6", + "refId": "build_node_duration", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "$task_duration - ($web_node_duration + $build_node_duration)", + "hide": false, + "refId": "Not accounted duration", + "type": "math" + } + ], + "title": "Diff", + "type": "stat" } ], "schemaVersion": 37, @@ -972,32 +1105,13 @@ "type": "postgres", "uid": "${DS_POSTGRESQL}" }, - "definition": "SELECT id \nFROM builds\nORDER BY id DESC;", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "build", - "options": [], - "query": "SELECT id \nFROM builds\nORDER BY id DESC;", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "current": {}, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "definition": "SELECT id from build_tasks where build_id = $build", + "definition": "SELECT id from build_tasks ORDER BY id", "hide": 0, "includeAll": false, "multi": false, "name": "build_task", "options": [], - "query": "SELECT id from build_tasks where build_id = $build", + "query": "SELECT id from build_tasks ORDER BY id", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -1014,6 +1128,6 @@ "timezone": "", "title": "Build task details", "uid": "vtQClqxVk", - "version": 19, + "version": 27, "weekStart": "" } \ No newline at end of file -- 2.43.5 From d1291c730871df13d83ed753403f55d22d8d17e2 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Wed, 8 Mar 2023 09:43:46 +0100 Subject: [PATCH 05/12] ALBS-1026 grafana-dashbords/Build analytics.json: added 95th percentile --- grafana-dashbords/Build analytics.json | 77 ++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json index 7de3172..ff04e6a 100644 --- a/grafana-dashbords/Build analytics.json +++ b/grafana-dashbords/Build analytics.json @@ -70,7 +70,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, @@ -859,6 +859,7 @@ "type": "postgres", "uid": "${DS_POSTGRESQL}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -880,7 +881,7 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 12, "x": 0, "y": 8 @@ -1005,9 +1006,44 @@ ], "limit": 50 } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" \nFROM \n (SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", + "refId": "95th", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], "title": "Build duration", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], "type": "stat" }, { @@ -1037,7 +1073,7 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 12, "x": 12, "y": 8 @@ -1171,6 +1207,35 @@ "limit": 50 }, "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4) as durations", + "refId": "95th percentile", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], "title": "Build task duration", @@ -1206,7 +1271,7 @@ "h": 8, "w": 24, "x": 0, - "y": 15 + "y": 13 }, "id": 20, "options": { @@ -1567,7 +1632,7 @@ "h": 15, "w": 24, "x": 0, - "y": 23 + "y": 21 }, "id": 2, "options": { @@ -1664,6 +1729,6 @@ "timezone": "", "title": "Build analytics", "uid": "02mg4oxVk", - "version": 21, + "version": 24, "weekStart": "" } \ No newline at end of file -- 2.43.5 From fafd7ae79bea4e3f3e82d333cde3266a04920e2f Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 9 Mar 2023 13:33:02 +0100 Subject: [PATCH 06/12] ALBS-1026 db.py: bug fix with updates grafana: added build task steps --- build_analytics/build_analytics/db.py | 11 +- grafana-dashbords/Build analytics.json | 3003 ++++++++++++++++++--- grafana-dashbords/Build details.json | 6 +- grafana-dashbords/Build task details.json | 566 ++-- 4 files changed, 2812 insertions(+), 774 deletions(-) diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index f2a12a1..68d6c62 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -151,9 +151,10 @@ class DB(): UPDATE web_node_stats SET start_ts = %s, end_ts = %s - WHERE build_task_id = %s; + WHERE build_task_id = %s AND stat_name_id = %s; ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) + cur.execute(sql, (stat.start_ts, stat.end_ts, + build_task.id, stat.stat_name_id)) # updating build_node_stats for stat in build_node_stats: @@ -161,9 +162,10 @@ class DB(): UPDATE build_node_stats SET start_ts = %s, end_ts = %s - WHERE build_task_id = %s; + WHERE build_task_id = %s and stat_name_id = %s; ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) + cur.execute(sql, (stat.start_ts, stat.end_ts, + build_task.id, stat.stat_name_id)) # commiting changes self.__conn.commit() @@ -177,7 +179,6 @@ class DB(): cur = self.__conn.cursor() cur.execute(sql) val = cur.fetchone() - print(val) if not val: return None return int(val[0]) diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json index ff04e6a..9c1e941 100644 --- a/grafana-dashbords/Build analytics.json +++ b/grafana-dashbords/Build analytics.json @@ -882,7 +882,7 @@ }, "gridPos": { "h": 5, - "w": 12, + "w": 9, "x": 0, "y": 8 }, @@ -1074,8 +1074,8 @@ }, "gridPos": { "h": 5, - "w": 12, - "x": 12, + "w": 9, + "x": 9, "y": 8 }, "id": 18, @@ -1246,38 +1246,73 @@ "type": "postgres", "uid": "${DS_POSTGRESQL}" }, - "description": "", "fieldConfig": { "defaults": { "color": { - "fixedColor": "#61a69d", - "mode": "fixed" + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null } ] }, "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "wait-time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 13 + "h": 5, + "w": 6, + "x": 18, + "y": 8 }, - "id": 20, + "id": 41, "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -1286,7 +1321,7 @@ "fields": "", "values": false }, - "showUnfilled": false + "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ @@ -1306,371 +1341,7 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", - "refId": "upload", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "start_ts", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "build_node_stats", - "urlPath": "" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", - "refId": "git_checkout", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", - "refId": "build_binaries", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", - "refId": "cas_notarize_artifacts", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", - "refId": "cas_source_authenticate", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", - "refId": "logs_processing", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "B", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Build task steps duration (SUM)", - "type": "bargauge" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "url" - }, - "properties": [ - { - "id": "custom.width", - "value": 287 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "custom.width", - "value": 107 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "hide": false, - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "rawSql": "SELECT\n bt.id, \n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) AND status_id = 2\nGROUP BY bt.id;", "refId": "A", "sql": { "columns": [ @@ -1692,29 +1363,2565 @@ "urlPath": "" } ], - "title": "Finished builds", + "title": "Build task wait time", "transformations": [ { - "id": "convertFieldType", + "id": "calculateField", "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "created" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} + "binary": { + "left": "web_node_duration", + "reducer": "sum", + "right": "build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": "task_duration", + "operator": "-", + "reducer": "sum", + "right": "web_node_duration build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "build_id": true, + "build_node_duration": true, + "id": true, + "task_duration": true, + "web_node_duration": true, + "web_node_duration build_node_duration": true + }, + "indexByName": {}, + "renameByName": { + "task_duration - web_node_duration build_node_duration": "wait-time" + } + } + }, + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "max", + "min", + "mean" + ] } } ], - "type": "table" + "type": "stat" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 26, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build_binaries" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "upload" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packages_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build_srpm" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "notarize_artifacts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "logs_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "source_authenticate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "git_checkout" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 33, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "packages_procession", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duration (%)", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "wait_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 14 + }, + "id": 20, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "packages_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + } + ], + "title": "Duration (absolute)", + "type": "bargauge" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest git_checkout step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build_task" + }, + "properties": [ + { + "id": "custom.width", + "value": 82 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 87 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 22 + }, + "id": 28, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "git_checkout", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest source_authenticate step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 3, + "y": 22 + }, + "id": 31, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "source_authenticate", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest build_binaries step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 6, + "y": 22 + }, + "id": 29, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "build_binaries", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest build_srpm step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 82 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 22 + }, + "id": 27, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "build_srpm", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest notarize_artifacts step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 12, + "y": 22 + }, + "id": 30, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "notarize_artifacts", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest logs_processing step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 15, + "y": 22 + }, + "id": 32, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "logs_processing", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest packages_processing step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 18, + "y": 22 + }, + "id": 34, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "packages_processing", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build_tasks with longest upload", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 79 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 21, + "y": 22 + }, + "id": 35, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\" , (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "upload", + "type": "table" + } + ], + "title": "Build Steps", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 24, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished builds", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "created" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "title": "Builds", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 39, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 103 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 108 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build id" + }, + "properties": [ + { + "id": "custom.width", + "value": 89 + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 37, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": " SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS task_duration\n FROM build_tasks AS bt\n JOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\n JOIN arch_enum on bt.arch_id = arch_enum.id \n WHERE $__unixEpochFilter(started_at)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT\n bt.id, \n bt.build_id,\n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) \nGROUP BY bt.id;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + }, + { + "id": "joinByField", + "options": { + "byField": "id", + "mode": "outer" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "(.*duration)|(wait time)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "arch" + }, + "properties": [ + { + "id": "custom.width", + "value": 111 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n" + } + ] + }, + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "task duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 43, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "wait time" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE $__unixEpochFilter(started_at) AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE $__unixEpochFilter(started_at) AND status_id = 2", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished builds tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "title": "Build tasks", + "type": "row" } ], - "refresh": "1m", + "refresh": false, "schemaVersion": 37, "style": "dark", "tags": [], @@ -1729,6 +3936,6 @@ "timezone": "", "title": "Build analytics", "uid": "02mg4oxVk", - "version": 24, + "version": 53, "weekStart": "" } \ No newline at end of file diff --git a/grafana-dashbords/Build details.json b/grafana-dashbords/Build details.json index d39e49a..c3d860b 100644 --- a/grafana-dashbords/Build details.json +++ b/grafana-dashbords/Build details.json @@ -161,7 +161,7 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id AND status_id >= 2;", + "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id;", "refId": "A", "sql": { "columns": [ @@ -183,7 +183,7 @@ "urlPath": "" } ], - "title": "Finished buld tasks", + "title": "Build tasks", "transformations": [ { "id": "convertFieldType", @@ -239,6 +239,6 @@ "timezone": "", "title": "Build details", "uid": "dmVtrz-4k", - "version": 5, + "version": 8, "weekStart": "" } \ No newline at end of file diff --git a/grafana-dashbords/Build task details.json b/grafana-dashbords/Build task details.json index 3edd794..a4c270a 100644 --- a/grafana-dashbords/Build task details.json +++ b/grafana-dashbords/Build task details.json @@ -41,12 +41,6 @@ "name": "PostgreSQL", "version": "1.0.0" }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, { "type": "panel", "id": "table", @@ -229,7 +223,23 @@ }, "unit": "s" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "wait time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 9, @@ -254,49 +264,6 @@ }, "pluginVersion": "9.3.2", "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0", - "refId": "upload", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "start_ts", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "build_node_stats", - "urlPath": "" - }, { "datasource": { "type": "postgres", @@ -306,8 +273,8 @@ "format": "table", "hide": false, "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2", - "refId": "build_srpm", + "rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3", + "refId": "git_checkout", "sql": { "columns": [ { @@ -335,8 +302,37 @@ "format": "table", "hide": false, "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3", - "refId": "git_checkout", + "rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2", + "refId": "build_srpm", "sql": { "columns": [ { @@ -413,35 +409,6 @@ "limit": 50 } }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;", - "refId": "cas_source_authenticate", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, { "datasource": { "type": "postgres", @@ -499,6 +466,146 @@ ], "limit": 50 } + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as build_node_task\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 6", + "refId": "build_node_task", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT end_ts - start_ts as web_node_task\nFROM web_node_stats\nWHERE build_task_id = $build_task and stat_name_id = 0", + "refId": "web_node_task", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT\n finished_at - started_at AS duration \nFROM build_tasks \nWHERE id = $build_task;", + "refId": "task_diration", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "$task_diration - $build_node_task - $web_node_task", + "hide": false, + "refId": "wait time", + "type": "math" } ], "title": "Task duration", @@ -605,7 +712,12 @@ }, "frameIndex": 1, "showHeader": true, - "sortBy": [] + "sortBy": [ + { + "desc": true, + "displayName": "started" + } + ] }, "pluginVersion": "9.3.2", "targets": [ @@ -690,8 +802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -811,287 +922,6 @@ } ], "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 6, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 1, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "finished" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", - "refId": "web node stats", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Build node stats", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 17, - "x": 3, - "y": 40 - }, - "id": 14, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n bt.finished_at - bt.started_at AS task_duration \nFROM build_tasks AS bt\nWHERE bt.id = $build_task;", - "refId": "task_duration", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT \n stats.end_ts - stats.start_ts AS web_node_duration\nFROM web_node_stats AS stats\nWHERE build_task_id = $build_task and stat_name_id=0;", - "refId": "web_node_duration", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT \n stats.end_ts - stats.start_ts AS build_node_duration\nFROM build_node_stats AS stats\nWHERE build_task_id = $build_task AND stat_name_id=6", - "refId": "build_node_duration", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "__expr__", - "uid": "${DS_EXPRESSION}" - }, - "expression": "$task_duration - ($web_node_duration + $build_node_duration)", - "hide": false, - "refId": "Not accounted duration", - "type": "math" - } - ], - "title": "Diff", - "type": "stat" } ], "schemaVersion": 37, @@ -1128,6 +958,6 @@ "timezone": "", "title": "Build task details", "uid": "vtQClqxVk", - "version": 27, + "version": 34, "weekStart": "" } \ No newline at end of file -- 2.43.5 From 03ed6e9c946f14ce762fd37292955730df382174 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 9 Mar 2023 13:41:49 +0100 Subject: [PATCH 07/12] added releases --- build_analytics/releases.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build_analytics/releases.txt b/build_analytics/releases.txt index bb25103..ff9e28f 100644 --- a/build_analytics/releases.txt +++ b/build_analytics/releases.txt @@ -1,6 +1,7 @@ 0.1.0 (2023-03-01) First version -0.2.0 -New parameter start_from -moved to double persition for timestamps \ No newline at end of file +0.2.0 (in progress) +- New parameter start_from +- Moved to double persition for timestamps +- Added metrics for build steps \ No newline at end of file -- 2.43.5 From b9bc5269ceea2a7c4c3ef1ba7d25f60461cd4cf5 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 9 Mar 2023 14:16:35 +0100 Subject: [PATCH 08/12] ALBS-1026 grafana/Build analytics: removed exluded/failed build tasks from duration calculation --- grafana-dashbords/Build analytics.json | 2944 ++---------------------- 1 file changed, 255 insertions(+), 2689 deletions(-) diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json index 9c1e941..5a0eb4c 100644 --- a/grafana-dashbords/Build analytics.json +++ b/grafana-dashbords/Build analytics.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_POSTGRESQL", - "label": "PostgreSQL", + "name": "DS_ALBS_ANALYTICS", + "label": "albs_analytics", "description": "", "type": "datasource", "pluginId": "postgres", @@ -11,17 +11,11 @@ ], "__elements": {}, "__requires": [ - { - "type": "panel", - "id": "bargauge", - "name": "Bar gauge", - "version": "" - }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.3.2" + "version": "9.3.6" }, { "type": "panel", @@ -93,7 +87,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "fieldConfig": { "defaults": { @@ -177,7 +171,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -206,7 +200,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -239,7 +233,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "description": "", "fieldConfig": { @@ -369,7 +363,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -398,7 +392,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -427,7 +421,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -456,7 +450,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -485,7 +479,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -518,7 +512,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "description": "", "fieldConfig": { @@ -708,7 +702,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -737,7 +731,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -766,7 +760,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -795,7 +789,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -824,7 +818,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "format": "table", @@ -857,9 +851,8 @@ { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -881,8 +874,8 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, + "h": 7, + "w": 12, "x": 0, "y": 8 }, @@ -901,13 +894,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -920,7 +913,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MAX(finished_at - created_at) AS MAX\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "max", + "refId": "MAX", "sql": { "columns": [ { @@ -944,7 +937,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -958,7 +951,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MIN(finished_at - created_at) AS MIN\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "min", + "refId": "MIN", "sql": { "columns": [ { @@ -979,16 +972,24 @@ "urlPath": "" }, { + "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], "format": "table", "hide": false, + "method": "GET", + "queryParams": "", "rawQuery": true, - "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", - "refId": "meidan", + "rawSql": "SELECT AVG(finished_at - created_at) AS AVG\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "AVG", "sql": { "columns": [ { @@ -1005,51 +1006,17 @@ } ], "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" \nFROM \n (SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", - "refId": "95th", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "urlPath": "" } ], "title": "Build duration", - "transformations": [ - { - "id": "organize", - "options": {} - } - ], "type": "stat" }, { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "description": "", "fieldConfig": { @@ -1073,9 +1040,9 @@ "overrides": [] }, "gridPos": { - "h": 5, - "w": 9, - "x": 9, + "h": 7, + "w": 12, + "x": 12, "y": 8 }, "id": 18, @@ -1093,13 +1060,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -1111,8 +1078,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", - "refId": "max", + "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id=2;", + "refId": "MAX", "sql": { "columns": [ { @@ -1136,7 +1103,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -1149,8 +1116,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", - "refId": "min", + "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2;", + "refId": "MIN", "sql": { "columns": [ { @@ -1174,7 +1141,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -1187,8 +1154,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4) as durations\n\n\n", - "refId": "median", + "rawSql": "SELECT AVG(finished_at - started_at) AS AVG\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id <> 2", + "refId": "AVG", "sql": { "columns": [ { @@ -1207,63 +1174,80 @@ "limit": 50 }, "urlPath": "" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4) as durations", - "refId": "95th percentile", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } } ], "title": "Build task duration", "type": "stat" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 5, + "panels": [], + "title": "Build details", + "type": "row" + }, { "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ + { + "color": "green" + }, { "color": "red", - "value": null + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 } ] }, - "unit": "s" - }, - "overrides": [ { "matcher": { "id": "byName", @@ -1271,65 +1255,43 @@ }, "properties": [ { - "id": "unit", - "value": "none" - }, - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "wait-time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } + "id": "custom.width", + "value": 107 } ] } ] }, "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 8 + "h": 15, + "w": 24, + "x": 0, + "y": 16 }, - "id": 41, + "id": 2, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "footer": { "fields": "", - "values": false + "reducer": [ + "sum" + ], + "show": false }, - "textMode": "auto" + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] }, - "pluginVersion": "9.3.2", + "pluginVersion": "9.3.6", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, "editorMode": "code", "fields": [ @@ -1338,10 +1300,11 @@ } ], "format": "table", + "hide": false, "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT\n bt.id, \n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) AND status_id = 2\nGROUP BY bt.id;", + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", "refId": "A", "sql": { "columns": [ @@ -1363,2565 +1326,168 @@ "urlPath": "" } ], - "title": "Build task wait time", + "title": "Finished builds", "transformations": [ { - "id": "calculateField", + "id": "convertFieldType", "options": { - "binary": { - "left": "web_node_duration", - "reducer": "sum", - "right": "build_node_duration" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "calculateField", - "options": { - "binary": { - "left": "task_duration", - "operator": "-", - "reducer": "sum", - "right": "web_node_duration build_node_duration" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "build_id": true, - "build_node_duration": true, - "id": true, - "task_duration": true, - "web_node_duration": true, - "web_node_duration build_node_duration": true - }, - "indexByName": {}, - "renameByName": { - "task_duration - web_node_duration build_node_duration": "wait-time" - } - } - }, - { - "id": "reduce", - "options": { - "labelsToFields": false, - "reducers": [ - "max", - "min", - "mean" - ] + "conversions": [ + { + "destinationType": "time", + "targetField": "created" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} } } ], - "type": "stat" + "type": "table" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 31 }, - "id": 26, - "panels": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#61a69d", - "mode": "fixed" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build_binaries" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "purple", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "upload" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "packages_processing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build_srpm" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "notarize_artifacts" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "logs_processing" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "source_authenticate" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "git_checkout" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "light-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 33, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", - "refId": "upload", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "start_ts", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "build_node_stats", - "urlPath": "" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", - "refId": "git_checkout", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", - "refId": "build_binaries", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", - "refId": "cas_notarize_artifacts", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", - "refId": "cas_source_authenticate", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", - "refId": "logs_processing", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "packages_procession", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Duration (%)", - "type": "piechart" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#61a69d", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "wait_time" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 14 - }, - "id": 20, - "options": { - "displayMode": "basic", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "vertical", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": false - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", - "refId": "git_checkout", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", - "refId": "cas_source_authenticate", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", - "refId": "build_binaries", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", - "refId": "cas_notarize_artifacts", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", - "refId": "logs_processing", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": false, - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", - "refId": "packages_processing", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - }, - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", - "refId": "upload", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "start_ts", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "build_node_stats", - "urlPath": "" - } - ], - "title": "Duration (absolute)", - "type": "bargauge" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest git_checkout step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build_task" - }, - "properties": [ - { - "id": "custom.width", - "value": 82 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 87 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 0, - "y": 22 - }, - "id": 28, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "git_checkout", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest source_authenticate step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 78 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 3, - "y": 22 - }, - "id": 31, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "source_authenticate", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest build_binaries step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 81 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 6, - "y": 22 - }, - "id": 29, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "build_binaries", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest build_srpm step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 81 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 82 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 9, - "y": 22 - }, - "id": 27, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "build_srpm", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest notarize_artifacts step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 78 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 12, - "y": 22 - }, - "id": 30, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "notarize_artifacts", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest logs_processing step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 78 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 15, - "y": 22 - }, - "id": 32, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "logs_processing", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build tasks with longest packages_processing step", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 78 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 18, - "y": 22 - }, - "id": 34, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "packages_processing", - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "Top 5 build_tasks with longest upload", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 79 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build task" - }, - "properties": [ - { - "id": "custom.width", - "value": 88 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 21, - "y": 22 - }, - "id": 35, - "options": { - "footer": { - "enablePagination": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT build_task_id AS \"build task\" , (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", - "refId": "build_srpm", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "upload", - "type": "table" - } - ], - "title": "Build Steps", + "id": 7, + "panels": [], + "title": "Build tasks details", "type": "row" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" }, - "id": 24, - "panels": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "url" - }, - "properties": [ - { - "id": "custom.width", - "value": 287 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "custom.width", - "value": 107 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" - } - ] - } - ] + "color": "green" } ] - }, - "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "hide": false, - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Finished builds", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "created" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - } - ], - "type": "table" - } - ], - "title": "Builds", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 39, - "panels": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - }, - { - "id": "custom.width", - "value": 103 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build_id" - }, - "properties": [ - { - "id": "custom.width", - "value": 88 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "name" - }, - "properties": [ - { - "id": "custom.width", - "value": 108 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build id" - }, - "properties": [ - { - "id": "custom.width", - "value": 89 - }, - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 37, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "hide": false, - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": " SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS task_duration\n FROM build_tasks AS bt\n JOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\n JOIN arch_enum on bt.arch_id = arch_enum.id \n WHERE $__unixEpochFilter(started_at)", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "format": "table", - "hide": true, - "rawQuery": true, - "rawSql": "SELECT\n bt.id, \n bt.build_id,\n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) \nGROUP BY bt.id;", - "refId": "B", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Build tasks", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - }, - { - "id": "joinByField", - "options": { - "byField": "id", - "mode": "outer" - } - } - ], - "type": "table" + } }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 15, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ { + "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_POSTGRESQL}" + "uid": "${DS_ALBS_ANALYTICS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "(.*duration)|(wait time)" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "custom.width", - "value": 70 - }, - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "arch" - }, - "properties": [ - { - "id": "custom.width", - "value": 111 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "build id" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "title": "", - "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n" - } - ] - }, - { - "id": "custom.width", - "value": 102 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "name" - }, - "properties": [ - { - "id": "custom.width", - "value": 75 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "task duration" - }, - "properties": [ - { - "id": "custom.width", - "value": 100 - } - ] - } - ] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 25 - }, - "id": 43, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "wait time" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ + "editorMode": "code", + "fields": [ { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE $__unixEpochFilter(started_at) AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE $__unixEpochFilter(started_at) AND status_id = 2", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" + "jsonPath": "" } ], - "title": "Finished builds tasks", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE $__unixEpochFilter(started_at) AND status_id >= 2;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" } - } - ], - "type": "table" + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" } ], - "title": "Build tasks", - "type": "row" + "title": "Finished buld tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" } ], - "refresh": false, + "refresh": "1m", "schemaVersion": 37, "style": "dark", "tags": [], @@ -3936,6 +1502,6 @@ "timezone": "", "title": "Build analytics", "uid": "02mg4oxVk", - "version": 53, + "version": 5, "weekStart": "" } \ No newline at end of file -- 2.43.5 From 22fb4a6423a34d3970dd325fafd36c49bbfd9881 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 10 Mar 2023 11:24:17 +0100 Subject: [PATCH 09/12] ALBS-1026 grafana: added dashboards for Build details, small fixes on other ones --- grafana-dashbords/Build analytics.json | 2954 +++++++++++++++++++-- grafana-dashbords/Build details.json | 2223 +++++++++++++++- grafana-dashbords/Build task details.json | 621 ++--- 3 files changed, 5210 insertions(+), 588 deletions(-) diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json index 5a0eb4c..374b650 100644 --- a/grafana-dashbords/Build analytics.json +++ b/grafana-dashbords/Build analytics.json @@ -1,8 +1,8 @@ { "__inputs": [ { - "name": "DS_ALBS_ANALYTICS", - "label": "albs_analytics", + "name": "DS_POSTGRESQL", + "label": "PostgreSQL", "description": "", "type": "datasource", "pluginId": "postgres", @@ -11,11 +11,17 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "9.3.6" + "version": "9.3.2" }, { "type": "panel", @@ -87,7 +93,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "fieldConfig": { "defaults": { @@ -171,13 +177,13 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS finished\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at is NOT NULL;", + "rawSql": "SELECT COUNT(*) AS finished\nFROM builds\nWHERE $__unixEpochFilter(created_at);", "refId": "B", "sql": { "columns": [ @@ -200,7 +206,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -233,7 +239,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -363,7 +369,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -392,7 +398,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -421,7 +427,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -450,7 +456,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -479,7 +485,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -512,7 +518,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "description": "", "fieldConfig": { @@ -630,7 +636,7 @@ { "id": "color", "value": { - "fixedColor": "red", + "fixedColor": "dark-orange", "mode": "fixed" } } @@ -702,7 +708,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -731,7 +737,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -760,7 +766,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -789,7 +795,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -818,7 +824,7 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "format": "table", @@ -851,8 +857,9 @@ { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -874,8 +881,8 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, + "h": 5, + "w": 9, "x": 0, "y": 8 }, @@ -894,13 +901,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -913,7 +920,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MAX(finished_at - created_at) AS MAX\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "MAX", + "refId": "max", "sql": { "columns": [ { @@ -937,7 +944,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -951,7 +958,7 @@ "queryParams": "", "rawQuery": true, "rawSql": "SELECT MIN(finished_at - created_at) AS MIN\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "MIN", + "refId": "min", "sql": { "columns": [ { @@ -972,24 +979,16 @@ "urlPath": "" }, { - "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], "format": "table", "hide": false, - "method": "GET", - "queryParams": "", "rawQuery": true, - "rawSql": "SELECT AVG(finished_at - created_at) AS AVG\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", - "refId": "AVG", + "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", + "refId": "meidan", "sql": { "columns": [ { @@ -1006,19 +1005,53 @@ } ], "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" }, - "urlPath": "" + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" \nFROM \n (SELECT (finished_at - created_at) AS duration \n FROM builds\n WHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL) as durations\n", + "refId": "95th", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], "title": "Build duration", + "transformations": [ + { + "id": "organize", + "options": {} + } + ], "type": "stat" }, { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, - "description": "", + "description": "Averages for build tasks with \"completed\"\nstatus", "fieldConfig": { "defaults": { "color": { @@ -1040,9 +1073,9 @@ "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, - "x": 12, + "h": 5, + "w": 9, + "x": 9, "y": 8 }, "id": 18, @@ -1060,13 +1093,13 @@ }, "textMode": "auto" }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1078,8 +1111,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id=2;", - "refId": "MAX", + "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2", + "refId": "max", "sql": { "columns": [ { @@ -1103,7 +1136,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1116,8 +1149,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2;", - "refId": "MIN", + "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2", + "refId": "min", "sql": { "columns": [ { @@ -1141,7 +1174,7 @@ "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1154,8 +1187,8 @@ "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT AVG(finished_at - started_at) AS AVG\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id <> 2", - "refId": "AVG", + "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2) as durations\n\n\n", + "refId": "median", "sql": { "columns": [ { @@ -1174,80 +1207,64 @@ "limit": 50 }, "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND status_id = 2) as durations", + "refId": "95th percentile", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], - "title": "Build task duration", + "title": "Build tasks duration (completed)", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 5, - "panels": [], - "title": "Build details", - "type": "row" - }, { "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, - "description": "", + "description": "How much time were spent in queue. \n\nStats for \"completed\" build tasks only", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "custom": { - "align": "center", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green" - }, { "color": "red", - "value": 80 + "value": null } ] - } + }, + "unit": "s" }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "url" - }, - "properties": [ - { - "id": "custom.width", - "value": 287 - } - ] - }, { "matcher": { "id": "byName", @@ -1255,43 +1272,65 @@ }, "properties": [ { - "id": "custom.width", - "value": 107 + "id": "unit", + "value": "none" + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "wait-time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } } ] } ] }, "gridPos": { - "h": 15, - "w": 24, - "x": 0, - "y": 16 + "h": 5, + "w": 6, + "x": 18, + "y": 8 }, - "id": 2, + "id": 41, "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "show": false + "fields": "", + "values": false }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration" - } - ] + "textMode": "auto" }, - "pluginVersion": "9.3.6", + "pluginVersion": "9.3.2", "targets": [ { "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, "editorMode": "code", "fields": [ @@ -1300,11 +1339,10 @@ } ], "format": "table", - "hide": false, "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "rawSql": "SELECT\n bt.id, \n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) AND status_id = 2\nGROUP BY bt.id;", "refId": "A", "sql": { "columns": [ @@ -1326,168 +1364,2568 @@ "urlPath": "" } ], - "title": "Finished builds", + "title": "Build task wait time (completed)", "transformations": [ { - "id": "convertFieldType", + "id": "calculateField", "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "created" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} + "binary": { + "left": "web_node_duration", + "reducer": "sum", + "right": "build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": "task_duration", + "operator": "-", + "reducer": "sum", + "right": "web_node_duration build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "build_id": true, + "build_node_duration": true, + "id": true, + "task_duration": true, + "web_node_duration": true, + "web_node_duration build_node_duration": true + }, + "indexByName": {}, + "renameByName": { + "task_duration - web_node_duration build_node_duration": "wait-time" + } + } + }, + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "max", + "min", + "mean" + ] } } ], - "type": "table" + "type": "stat" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 13 }, - "id": 7, - "panels": [], - "title": "Build tasks details", + "id": 26, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build_binaries" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "upload" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packages_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build_srpm" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "notarize_artifacts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "logs_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "source_authenticate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "git_checkout" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 33, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "packages_procession", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duration (%)", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "wait_time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 14 + }, + "id": 20, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as git_checkout\nFROM build_node_stats \nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts);", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as source_authenticate\nFROM build_node_stats \nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts);", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_srpm\nFROM build_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as build_binaries\nFROM build_node_stats \nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts);", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats \nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as logs_processing\nFROM web_node_stats \nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts);", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as packages_processing\nFROM web_node_stats \nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts);", + "refId": "packages_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT SUM(end_ts - start_ts) as upload\nFROM build_node_stats \nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts);", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + } + ], + "title": "Duration (absolute)", + "type": "bargauge" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest git_checkout step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build_task" + }, + "properties": [ + { + "id": "custom.width", + "value": 82 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 87 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 22 + }, + "id": 28, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 3 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "git_checkout", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest source_authenticate step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 3, + "y": 22 + }, + "id": 31, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 8 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "source_authenticate", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest build_binaries step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 6, + "y": 22 + }, + "id": 29, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 4 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "build_binaries", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest build_srpm step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 82 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 22 + }, + "id": 27, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "build_srpm", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest notarize_artifacts step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 12, + "y": 22 + }, + "id": 30, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 7 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "notarize_artifacts", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest logs_processing step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 15, + "y": 22 + }, + "id": 32, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 1 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "logs_processing", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build tasks with longest packages_processing step", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 18, + "y": 22 + }, + "id": 34, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\", (end_ts - start_ts) as duration\nFROM web_node_stats\nWHERE stat_name_id = 2 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "packages_processing", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Top 5 build_tasks with longest upload", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 79 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build task" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 21, + "y": 22 + }, + "id": 35, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT build_task_id AS \"build task\" , (end_ts - start_ts) as duration\nFROM build_node_stats\nWHERE stat_name_id = 0 AND $__unixEpochFilter(start_ts)\nORDER BY duration DESC\nLIMIT 5;", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "upload", + "type": "table" + } + ], + "title": "Build Steps", "type": "row" }, { - "datasource": { - "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "displayMode": "auto", - "filterable": true, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - } - ] - }, + "collapsed": true, "gridPos": { - "h": 12, + "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 14 }, - "id": 15, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "duration" - } - ] - }, - "pluginVersion": "9.3.6", - "targets": [ + "id": 24, + "panels": [ { - "cacheDurationSeconds": 300, "datasource": { "type": "postgres", - "uid": "${DS_ALBS_ANALYTICS}" + "uid": "${DS_POSTGRESQL}" }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE $__unixEpochFilter(started_at) AND status_id >= 2;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "groupBy": [ + }, + "overrides": [ { - "property": { - "type": "string" + "matcher": { + "id": "byName", + "options": "duration" }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Finished buld tasks", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" + "properties": [ + { + "id": "unit", + "value": "s" + } + ] }, { - "destinationType": "time", - "targetField": "finished" + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" + } + ] + } + ] } - ], - "fields": {} - } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished builds", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "created" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" } ], - "type": "table" + "title": "Builds", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 39, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 103 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 88 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 108 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build id" + }, + "properties": [ + { + "id": "custom.width", + "value": 89 + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 37, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "task_duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": " SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS task_duration\n FROM build_tasks AS bt\n JOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\n JOIN arch_enum on bt.arch_id = arch_enum.id \n WHERE $__unixEpochFilter(started_at)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": true, + "rawQuery": true, + "rawSql": "SELECT\n bt.id, \n bt.build_id,\n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) \nGROUP BY bt.id;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "All build tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + }, + { + "id": "joinByField", + "options": { + "byField": "id", + "mode": "outer" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "(.*duration)|(wait time)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "arch" + }, + "properties": [ + { + "id": "custom.width", + "value": 111 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n" + } + ] + }, + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "task duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 43, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "task duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE $__unixEpochFilter(started_at) AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE $__unixEpochFilter(started_at) AND status_id = 2", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Completed build tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "title": "Build tasks", + "type": "row" } ], - "refresh": "1m", + "refresh": false, "schemaVersion": 37, "style": "dark", "tags": [], @@ -1502,6 +3940,6 @@ "timezone": "", "title": "Build analytics", "uid": "02mg4oxVk", - "version": 5, + "version": 63, "weekStart": "" } \ No newline at end of file diff --git a/grafana-dashbords/Build details.json b/grafana-dashbords/Build details.json index c3d860b..24c67db 100644 --- a/grafana-dashbords/Build details.json +++ b/grafana-dashbords/Build details.json @@ -11,18 +11,36 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.3.2" }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, { "type": "datasource", "id": "postgres", "name": "PostgreSQL", "version": "1.0.0" }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, { "type": "panel", "id": "table", @@ -59,6 +77,19 @@ "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "Summary", + "type": "row" + }, { "datasource": { "type": "postgres", @@ -83,29 +114,15 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": false, - "title": "", - "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" - } - ] - } - ] - }, { "matcher": { "id": "byName", @@ -117,16 +134,57 @@ "value": "s" } ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}" + } + ] + } + ] } ] }, "gridPos": { - "h": 14, + "h": 3, "w": 24, "x": 0, - "y": 0 + "y": 1 }, - "id": 2, + "id": 10, "options": { "footer": { "fields": "", @@ -139,7 +197,7 @@ "sortBy": [ { "desc": true, - "displayName": "build_id" + "displayName": "duration" } ] }, @@ -158,10 +216,11 @@ } ], "format": "table", + "hide": false, "method": "GET", "queryParams": "", "rawQuery": true, - "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id;", + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE id = $build_id ", "refId": "A", "sql": { "columns": [ @@ -183,7 +242,7 @@ "urlPath": "" } ], - "title": "Build tasks", + "title": "Build info", "transformations": [ { "id": "convertFieldType", @@ -191,7 +250,7 @@ "conversions": [ { "destinationType": "time", - "targetField": "started" + "targetField": "created" }, { "destinationType": "time", @@ -203,8 +262,2116 @@ } ], "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 0, + "y": 4 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS idle FROM build_tasks \nWHERE build_id = $build_id AND status_id = 0;", + "refId": "idle", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS started FROM build_tasks \nWHERE build_id = $build_id AND status_id = 1;", + "refId": "started", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS completed FROM build_tasks \nWHERE build_id = $build_id AND status_id = 2;", + "refId": "completed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS failed FROM build_tasks \nWHERE build_id = $build_id AND status_id = 3;", + "refId": "failed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS excluded FROM build_tasks \nWHERE build_id = $build_id AND status_id = 4;", + "refId": "excluded", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by status", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "x86_64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "i686" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "aarch64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ppc64le" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 7, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS i686 FROM build_tasks \nWHERE build_id = $build_id AND arch_id = 0;", + "refId": "i686", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS x86_64 FROM build_tasks \nWHERE build_id = $build_id AND arch_id = 1;", + "refId": "x86_64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS aarch64 FROM build_tasks \nWHERE build_id = $build_id AND arch_id = 2;", + "refId": "aarch64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS ppc64le FROM build_tasks \nWHERE build_id = $build_id AND arch_id = 3;", + "refId": "ppc64le", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS s390x FROM build_tasks \nWHERE build_id = $build_id AND arch_id = 4;", + "refId": "s390x", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by arch", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "Averages for build tasks with \"completed\"\nstatus", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 14, + "y": 4 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE build_id = $build_id AND status_id = 2", + "refId": "max", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE build_id = $build_id AND status_id = 2", + "refId": "min", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY durations.duration) as median FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE build_id = $build_id AND status_id = 2) as durations\n\n\n", + "refId": "median", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY durations.duration) as \"95th percentile\" FROM \n(SELECT (finished_at - started_at) AS duration\nFROM build_tasks\nWHERE build_id = $build_id AND status_id = 2) as durations", + "refId": "95th percentile", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks duration (completed)", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "How much time were spent in queue. \n\nStats for \"completed\" build tasks only", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "wait-time" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 10, + "x": 14, + "y": 8 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id, \n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE build_id = $build_id AND status_id = 2\nGROUP BY bt.id;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build task wait time (completed)", + "transformations": [ + { + "id": "calculateField", + "options": { + "binary": { + "left": "web_node_duration", + "reducer": "sum", + "right": "build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": "task_duration", + "operator": "-", + "reducer": "sum", + "right": "web_node_duration build_node_duration" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "build_id": true, + "build_node_duration": true, + "id": true, + "task_duration": true, + "web_node_duration": true, + "web_node_duration build_node_duration": true + }, + "indexByName": {}, + "renameByName": { + "task_duration - web_node_duration build_node_duration": "wait-time" + } + } + }, + { + "id": "reduce", + "options": { + "labelsToFields": false, + "reducers": [ + "max", + "min", + "mean" + ] + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 23, + "panels": [], + "title": "Build steps", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "build_binaries" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "upload" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "packages_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build_srpm" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "notarize_artifacts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "logs_processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "source_authenticate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "git_checkout" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as upload\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 0 AND build_tasks.build_id = $build_id ", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as build_srpm\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as git_checkout\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 3 AND build_tasks.build_id = $build_id ", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as build_binaries\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 4 AND build_tasks.build_id = $build_id ", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 7 AND build_tasks.build_id = $build_id ", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as source_authenticate\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 8 AND build_tasks.build_id = $build_id ", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n SUM(end_ts - start_ts) AS logs_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", + "refId": "packages_procession", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duration (%)", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#61a69d", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 13 + }, + "id": 21, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as upload\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 0 AND build_tasks.build_id = $build_id ", + "refId": "upload", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "start_ts", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "build_node_stats", + "urlPath": "" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as build_srpm\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ", + "refId": "build_srpm", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as git_checkout\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 3 AND build_tasks.build_id = $build_id ", + "refId": "git_checkout", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as build_binaries\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 4 AND build_tasks.build_id = $build_id ", + "refId": "build_binaries", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as notarize_artifacts\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 7 AND build_tasks.build_id = $build_id ", + "refId": "cas_notarize_artifacts", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n sum(end_ts - start_ts) as source_authenticate\nFROM build_node_stats bs\nJOIN build_tasks \n on bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 8 AND build_tasks.build_id = $build_id ", + "refId": "cas_source_authenticate", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n SUM(end_ts - start_ts) AS logs_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", + "refId": "logs_processing", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ", + "refId": "packages_procession", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duration (absolute)", + "type": "bargauge" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 18, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "All build tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "(.*duration)|(wait time)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + }, + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "arch" + }, + "properties": [ + { + "id": "custom.width", + "value": 111 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "build id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n" + } + ] + }, + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "task duration" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 25, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "task duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE bt.build_id = $build_id AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE bt.build_id = $build_id AND status_id = 2", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Completed build tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "title": "Build tasks", + "type": "row" } ], + "refresh": false, "schemaVersion": 37, "style": "dark", "tags": [], @@ -232,13 +2399,13 @@ ] }, "time": { - "from": "now-6h", - "to": "now" + "from": "2023-03-09T18:59:42.117Z", + "to": "2023-03-10T18:59:42.117Z" }, "timepicker": {}, "timezone": "", "title": "Build details", "uid": "dmVtrz-4k", - "version": 8, + "version": 21, "weekStart": "" } \ No newline at end of file diff --git a/grafana-dashbords/Build task details.json b/grafana-dashbords/Build task details.json index a4c270a..2249199 100644 --- a/grafana-dashbords/Build task details.json +++ b/grafana-dashbords/Build task details.json @@ -119,6 +119,23 @@ "value": "s" } ] + }, + { + "matcher": { + "id": "byName", + "options": "build_id" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n" + } + ] + } + ] } ] }, @@ -612,7 +629,7 @@ "type": "bargauge" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -620,308 +637,308 @@ "y": 12 }, "id": 8, - "panels": [], + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "finished" + }, + "properties": [ + { + "id": "custom.width", + "value": 235 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "started" + }, + "properties": [ + { + "id": "custom.width", + "value": 205 + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "started" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", + "refId": "web node stats", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Web node stats", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "started" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_POSTGRESQL}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", + "refId": "web node stats", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build node stats", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], "title": "Details", "type": "row" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "finished" - }, - "properties": [ - { - "id": "custom.width", - "value": 235 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "started" - }, - "properties": [ - { - "id": "custom.width", - "value": 205 - } - ] - } - ] - }, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 1, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "started" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", - "refId": "web node stats", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Web node stats", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - } - ], - "type": "table" - }, - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "filterable": true, - "inspect": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "duration" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "started" - }, - "properties": [ - { - "id": "custom.width", - "value": 200 - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 1, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task", - "refId": "web node stats", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Build node stats", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "started" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - } - ], - "type": "table" } ], "schemaVersion": 37, @@ -958,6 +975,6 @@ "timezone": "", "title": "Build task details", "uid": "vtQClqxVk", - "version": 34, + "version": 37, "weekStart": "" } \ No newline at end of file -- 2.43.5 From d54ab39941f9657921e534aa33086e52616704ee Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 10 Mar 2023 11:41:01 +0100 Subject: [PATCH 10/12] db: bugfix with closed connection on subsequent srapping --- build_analytics/build_analytics/extractor/start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_analytics/build_analytics/extractor/start.py b/build_analytics/build_analytics/extractor/start.py index 90f59c1..d7f86e3 100644 --- a/build_analytics/build_analytics/extractor/start.py +++ b/build_analytics/build_analytics/extractor/start.py @@ -64,12 +64,13 @@ def start(yml_path: str): logging.error('unsupported DB schema: want %s, have %s', DB_SCHEMA_VER, cur_version) sys.exit(1) + db.close_conn() while True: logging.info('Starting extraction proccess') api = APIclient(api_root=config.albs_url, jwt=config.jwt, timeout=config.api_timeout) - + db = DB(config.db_config) extractor = Extractor(config, api, db) logging.info('Starting builds insertion') -- 2.43.5 From 3ed0dcdd786bd2db042917daeb5255b3250807cc Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 10 Mar 2023 19:35:12 +0100 Subject: [PATCH 11/12] added build step: multilib_processing db: stop adding empty build steps (without start_ts) --- build_analytics/build_analytics/api_client.py | 3 +- build_analytics/build_analytics/const.py | 1 + build_analytics/build_analytics/db.py | 83 +++++++++++++++---- .../build_analytics/models/web_node_stats.py | 1 + build_analytics/migrations/2.sql | 13 +++ 5 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 build_analytics/migrations/2.sql diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index f30adbd..56a293b 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -72,7 +72,8 @@ class APIclient(): return build_node_stats def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats: - keys = ['build_done', 'logs_processing', 'packages_processing'] + keys = ['build_done', 'logs_processing', + 'packages_processing', 'multilib_processing'] params = {} logging.debug('raw json: %s', stats) for k in keys: diff --git a/build_analytics/build_analytics/const.py b/build_analytics/build_analytics/const.py index c29d1d9..c9c9370 100644 --- a/build_analytics/build_analytics/const.py +++ b/build_analytics/build_analytics/const.py @@ -27,6 +27,7 @@ class WebNodeStatsEnum(IntEnum): build_done = 0 logs_processing = 1 packages_processing = 2 + multilib_processing = 3 class BuildNodeStatsEnum(IntEnum): diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index 68d6c62..b266249 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -50,6 +50,11 @@ class DB(): # inserting web node stats for stat in web_node_stats: + + # do not insert empty stats + if stat.start_ts is None: + continue + sql = ''' INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts) VALUES (%s, %s, %s, %s); @@ -57,10 +62,15 @@ class DB(): cur.execute(sql, (stat.build_task_id, stat.stat_name_id, stat.start_ts, stat.end_ts)) logging.debug('raw SQL query: %s', cur.query) + self.__conn.commit() # inserting build node stats for stat in build_node_stats: - logging.debug('BuildNodeStats: %s', stat) + + # do not insert empty stats + if stat.start_ts is None: + continue + sql = ''' INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) VALUES (%s, %s, %s, %s); @@ -134,7 +144,6 @@ class DB(): build_node_stats: List[BuildNodeStatDB]): cur = self.__conn.cursor() - # updating build_task sql = ''' UPDATE build_tasks SET status_id = %s, @@ -144,28 +153,55 @@ class DB(): ''' cur.execute(sql, (build_task.status_id, build_task.started_at, build_task.finished_at, build_task.id)) + logging.debug('raw SQL query: %s', cur.query) # updating web_node_stats for stat in web_node_stats: - sql = ''' - UPDATE web_node_stats - SET start_ts = %s, - end_ts = %s - WHERE build_task_id = %s AND stat_name_id = %s; - ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, - build_task.id, stat.stat_name_id)) + logging.debug( + 'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id) + if self.stat_exists(build_task_id=stat.build_task_id, + stat_name_id=stat.stat_name_id, + table_name='web_node_stats'): + sql = ''' + UPDATE web_node_stats + SET start_ts = %(start_ts)s, end_ts = %(end_ts)s + WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s + ''' + else: + sql = ''' + INSERT INTO web_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s); + ''' + params = {'build_task_id': build_task.id, + 'stat_name_id': stat.stat_name_id, + 'start_ts': stat.start_ts, + 'end_ts': stat.end_ts} + cur.execute(sql, params) + logging.debug('raw SQL query: %s', cur.query) # updating build_node_stats for stat in build_node_stats: - sql = ''' - UPDATE build_node_stats - SET start_ts = %s, - end_ts = %s - WHERE build_task_id = %s and stat_name_id = %s; - ''' - cur.execute(sql, (stat.start_ts, stat.end_ts, - build_task.id, stat.stat_name_id)) + logging.debug( + 'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id) + if self.stat_exists(build_task_id=stat.build_task_id, + stat_name_id=stat.stat_name_id, + table_name='build_node_stats'): + sql = ''' + UPDATE build_node_stats + SET start_ts = %(start_ts)s, end_ts = %(end_ts)s + WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s + ''' + else: + sql = ''' + INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s); + ''' + params = {'build_task_id': build_task.id, + 'stat_name_id': stat.stat_name_id, + 'start_ts': stat.start_ts, + 'end_ts': stat.end_ts} + logging.debug('raw SQL query: %s', cur.query) + cur.execute(sql, params) # commiting changes self.__conn.commit() @@ -182,3 +218,14 @@ class DB(): if not val: return None return int(val[0]) + + def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool: + sql = f''' + SELECT COUNT(build_task_id) + FROM {table_name} + WHERE build_task_id = %s AND stat_name_id = %s; + ''' + cur = self.__conn.cursor() + cur.execute(sql, (build_task_id, stat_name_id)) + val = int(cur.fetchone()[0]) + return val == 1 diff --git a/build_analytics/build_analytics/models/web_node_stats.py b/build_analytics/build_analytics/models/web_node_stats.py index 0476863..504ccea 100644 --- a/build_analytics/build_analytics/models/web_node_stats.py +++ b/build_analytics/build_analytics/models/web_node_stats.py @@ -15,6 +15,7 @@ class WebNodeStats(BaseModel): build_done: BuildStat logs_processing: BuildStat packages_processing: BuildStat + multilib_processing: BuildStat def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]: result = [] diff --git a/build_analytics/migrations/2.sql b/build_analytics/migrations/2.sql new file mode 100644 index 0000000..1b10c5f --- /dev/null +++ b/build_analytics/migrations/2.sql @@ -0,0 +1,13 @@ +BEGIN; + +INSERT INTO web_node_stats_enum (id, value) +VALUES + (3, 'multilib_processing'); + +ALTER TABLE web_node_stats +ADD CONSTRAINT web_node_stats_unique UNIQUE (build_task_id, stat_name_id); + +ALTER TABLE build_node_stats +ADD CONSTRAINT build_node_stats_unique UNIQUE (build_task_id, stat_name_id); + +COMMIT; \ No newline at end of file -- 2.43.5 From 4c843a27c6b552226262ac9c3c713013330e3f70 Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Fri, 10 Mar 2023 19:44:54 +0100 Subject: [PATCH 12/12] db: forgot to update schema version --- build_analytics/build_analytics/const.py | 2 +- build_analytics/migrations/2.sql | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/build_analytics/build_analytics/const.py b/build_analytics/build_analytics/const.py index c9c9370..dd79f1c 100644 --- a/build_analytics/build_analytics/const.py +++ b/build_analytics/build_analytics/const.py @@ -3,7 +3,7 @@ from enum import IntEnum # supported schema version -DB_SCHEMA_VER = 1 +DB_SCHEMA_VER = 2 # ENUMS diff --git a/build_analytics/migrations/2.sql b/build_analytics/migrations/2.sql index 1b10c5f..160451b 100644 --- a/build_analytics/migrations/2.sql +++ b/build_analytics/migrations/2.sql @@ -10,4 +10,8 @@ ADD CONSTRAINT web_node_stats_unique UNIQUE (build_task_id, stat_name_id); ALTER TABLE build_node_stats ADD CONSTRAINT build_node_stats_unique UNIQUE (build_task_id, stat_name_id); + +UPDATE schema_version +SET version = 2; + COMMIT; \ No newline at end of file -- 2.43.5