diff --git a/build_analitycs/build_analytics/models/enums.py b/build_analitycs/build_analytics/models/enums.py deleted file mode 100644 index 5b18fdf..0000000 --- a/build_analitycs/build_analytics/models/enums.py +++ /dev/null @@ -1,17 +0,0 @@ -from enum import IntEnum - - -class ArchEnum(IntEnum): - i686 = 0 - x86_64 = 1 - aarch64 = 2 - ppc64le = 3 - s390x = 4 - - -class BuildTaskEnum(IntEnum): - idle = 0 - started = 1 - completed = 2 - failed = 3 - excluded = 4 diff --git a/build_analitycs/releases.txt b/build_analitycs/releases.txt deleted file mode 100644 index 416adfb..0000000 --- a/build_analitycs/releases.txt +++ /dev/null @@ -1,2 +0,0 @@ -0.1.0 (2023-03-01) -First version \ No newline at end of file diff --git a/build_analitycs/__init__.py b/build_analytics/__init__.py similarity index 100% rename from build_analitycs/__init__.py rename to build_analytics/__init__.py diff --git a/build_analitycs/build_analytics/__init__.py b/build_analytics/build_analytics/__init__.py similarity index 100% rename from build_analitycs/build_analytics/__init__.py rename to build_analytics/build_analytics/__init__.py diff --git a/build_analitycs/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py similarity index 54% rename from build_analitycs/build_analytics/api_client.py rename to build_analytics/build_analytics/api_client.py index bcc60ad..c72dca8 100644 --- a/build_analitycs/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -1,14 +1,15 @@ from datetime import datetime import logging - from urllib.parse import urljoin from typing import Dict, List +import requests from .models.build import Build from .models.build_task import BuildTask - -import requests +from .models.build_node_stats import BuildNodeStats +from .models.build_stat import BuildStat +from .models.web_node_stats import WebNodeStats TZ_OFFSET = '+00:00' @@ -51,23 +52,61 @@ class APIclient(): response.raise_for_status() return self._parse_build(response.json()) + def __parse_build_node_stats(self, stats: Dict) -> BuildNodeStats: + + keys = ['build_all', 'build_binaries', 'build_packages', 'build_srpm', 'build_node_task', + 'cas_notarize_artifacts', 'cas_source_authenticate', 'git_checkout', 'upload'] + params = {} + for k in keys: + try: + params[k] = BuildStat( + start_ts=datetime.fromisoformat( + stats[k]['start_ts']+TZ_OFFSET) if stats[k]['start_ts'] else None, + end_ts=datetime.fromisoformat( + stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) + except KeyError: + params[k] = BuildStat() + return BuildNodeStats(**params) + + def __parse_web_node_stats(self, stats: Dict) -> WebNodeStats: + keys = ['build_done', 'logs_processing', 'packages_processing'] + params = {} + for k in keys: + try: + params[k] = BuildStat( + start_ts=datetime.fromisoformat( + stats[k]['start_ts']+TZ_OFFSET) if stats[k]['start_ts'] else None, + end_ts=datetime.fromisoformat( + stats[k]['end_ts']+TZ_OFFSET) if stats[k]['end_ts'] else None) + except KeyError: + params[k] = BuildStat() + return WebNodeStats(**params) + def _parse_build_tasks(self, tasks_json: Dict, build_id: int) -> List[BuildTask]: result = [] for task in tasks_json: try: started_at = datetime.fromisoformat( - task['started_at']+TZ_OFFSET) \ - if task['started_at'] else None - finished_at = datetime.fromisoformat(task['finished_at']+TZ_OFFSET) \ - if task['finished_at'] else None + task['started_at']+TZ_OFFSET) if task['started_at'] else None + finished_at = datetime.fromisoformat( + task['finished_at']+TZ_OFFSET) if task['finished_at'] else None name = task['ref']['url'].split('/')[-1].replace('.git', '') + if not task['performance_stats']: + logging.warning( + "no perfomance_stats for build_id: %s, build_task_id: %s", build_id, task['id']) + stats = {'build_node_stats': {}, 'build_done_stats': {}} + else: + stats = task['performance_stats'][0]['statistics'] + params = {'id': task['id'], 'name': name, 'build_id': build_id, 'started_at': started_at, 'finished_at': finished_at, 'arch': task['arch'], - 'status_id': task['status']} + 'status_id': task['status'], + 'build_node_stats': self.__parse_build_node_stats(stats['build_node_stats']), + 'web_node_stats': self.__parse_web_node_stats(stats['build_done_stats'])} result.append(BuildTask(**params)) except Exception as err: # pylint: disable=broad-except logging.error("Cant convert build_task JSON %s (build_id %s) to BuildTask model: %s", @@ -79,8 +118,8 @@ class APIclient(): def _parse_build(self, build_json: Dict) -> Build: url = f"https://build.almalinux.org/build/{build_json['id']}" created_at = datetime.fromisoformat(build_json['created_at']+TZ_OFFSET) - finished_at = datetime.fromisoformat(build_json['finished_at']+TZ_OFFSET) \ - if build_json['finished_at'] else None + finished_at = datetime.fromisoformat( + build_json['finished_at']+TZ_OFFSET) if build_json['finished_at'] else None build_tasks = self._parse_build_tasks( build_json['tasks'], build_json['id']) diff --git a/build_analitycs/build_analytics/db.py b/build_analytics/build_analytics/db.py similarity index 62% rename from build_analitycs/build_analytics/db.py rename to build_analytics/build_analytics/db.py index 3c5e948..dac6d6a 100644 --- a/build_analitycs/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -1,11 +1,13 @@ from datetime import datetime -from typing import Union, Dict +from typing import Union, Dict, List import psycopg2 from .models.build_db import BuildDB from .models.build_task_db import BuildTaskDB +from .models.build_node_stat_db import BuildNodeStatDB from .models.db_config import DbConfig +from .models.web_node_stat_db import WebNodeStatDB class DB(): @@ -33,15 +35,37 @@ class DB(): build.created_at, build.finished_at)) self.__conn.commit() - def insert_buildtask(self, build_task: BuildTaskDB): + def insert_buildtask(self, build_task: BuildTaskDB, web_node_stats: List[WebNodeStatDB], + build_node_stats: List[BuildNodeStatDB]): + + cur = self.__conn.cursor() + # inserting build_task sql = ''' INSERT INTO build_tasks(id, name, build_id, arch_id, started_at, finished_at, status_id) VALUES (%s, %s, %s, %s, %s, %s, %s); ''' - - cur = self.__conn.cursor() cur.execute(sql, (build_task.id, build_task.name, build_task.build_id, build_task.arch_id, build_task.started_at, build_task.finished_at, build_task.status_id)) + + # inserting web node stats + for stat in web_node_stats: + sql = ''' + INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%s, %s, %s, %s); + ''' + cur.execute(sql, (stat.build_task_id, stat.stat_name_id, + stat.start_ts, stat.end_ts)) + + # inserting build node stats + for stat in build_node_stats: + sql = ''' + INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts) + VALUES (%s, %s, %s, %s); + ''' + cur.execute(sql, (stat.build_task_id, stat.stat_name_id, + stat.start_ts, stat.end_ts)) + + # commiting changes self.__conn.commit() def get_latest_build_id(self) -> Union[int, None]: @@ -101,7 +125,12 @@ class DB(): cur.execute(sql, (build.finished_at, build.id)) self.__conn.commit() - def update_build_task(self, build: BuildTaskDB): + def update_build_task(self, build_task: BuildTaskDB, + web_node_stats: List[WebNodeStatDB], + build_node_stats: List[BuildNodeStatDB]): + cur = self.__conn.cursor() + + # updating build_task sql = ''' UPDATE build_tasks SET status_id = %s, @@ -109,7 +138,28 @@ class DB(): finished_at = %s WHERE id = %s; ''' - cur = self.__conn.cursor() - cur.execute(sql, (build.status_id, build.started_at, - build.finished_at, build.id)) + cur.execute(sql, (build_task.status_id, build_task.started_at, + build_task.finished_at, build_task.id)) + + # updating web_node_stats + for stat in web_node_stats: + sql = ''' + UPDATE web_node_stats + SET start_ts = %s, + end_ts = %s + WHERE build_task_id = %s; + ''' + cur.execute(sql, (stat.start_ts, stat.end_ts)) + + # updating build_node_stats + for stat in build_node_stats: + sql = ''' + UPDATE build_node_stats + SET start_ts = %s, + end_ts = %s, + WHERE build_task_id = %s; + ''' + cur.execute(sql, (stat.start_ts, stat.end_ts)) + + # commiting changes self.__conn.commit() diff --git a/build_analitycs/build_analytics/extractor/__jnit__.py b/build_analytics/build_analytics/extractor/__jnit__.py similarity index 100% rename from build_analitycs/build_analytics/extractor/__jnit__.py rename to build_analytics/build_analytics/extractor/__jnit__.py diff --git a/build_analitycs/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py similarity index 59% rename from build_analitycs/build_analytics/extractor/extractor.py rename to build_analytics/build_analytics/extractor/extractor.py index 82b4b9b..f210692 100644 --- a/build_analitycs/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -12,6 +12,7 @@ from ..api_client import APIclient class Extractor: def __init__(self, config: ExtractorConfig, api: APIclient, db: DB): + self.start_from = config.start_from self.oldest_build_age = config.oldest_build_age self.api = api self.db = db @@ -21,7 +22,7 @@ class Extractor: page_num = 1 last_build_id = self.db.get_latest_build_id() if not last_build_id: - last_build_id = 0 + last_build_id = self.start_from logging.info("last_build_id: %s", last_build_id) stop = False @@ -34,11 +35,25 @@ class Extractor: stop = True break - # inserting build and build tasks + # inserting build build tasks and build tasks statistics logging.info("inserting %s", build.id) - self.db.insert_build(build.as_db_model()) + try: + self.db.insert_build(build.as_db_model()) + except Exception as error: # pylint: disable=broad-except + logging.error('failed to insert build %d: %s', + build.id, error, exc_info=True) + continue + for build_task in build.build_tasks: - self.db.insert_buildtask(build_task.as_db_model()) + try: + self.db.insert_buildtask(build_task.as_db_model(), + build_task.web_node_stats.as_db_model( + build_task.id), + build_task.web_node_stats.as_db_model( + build_task.id)) + except Exception as error: # pylint: disable=broad-except + logging.error('failed to insert build task %d: %s', + build_task.id, error, exc_info=True) build_count += 1 page_num += 1 return build_count @@ -49,25 +64,31 @@ class Extractor: removed_count = self.db.cleanup_builds(self.oldest_build_age) logging.info('removed %d entries', removed_count) - def __update_build_tasks_statuses(self, build_tasks: List[BuildTask], - build_tasks_status_db: Dict[int, int]): + def __update_build_tasks(self, build_tasks: List[BuildTask], + build_tasks_status_db: Dict[int, int]): for b in build_tasks: if b.status_id != build_tasks_status_db[b.id]: - logging.info('build taks %d status have changed %s -> %s. Updating DB', + logging.info('build: %s, build task %d status have changed %s -> %s. Updating DB', + b.build_id, b.id, BuildTaskEnum( build_tasks_status_db[b.id]).name, BuildTaskEnum(b.status_id).name) try: - self.db.update_build_task(b.as_db_model()) + self.db.update_build_task(b.as_db_model(), + b.web_node_stats.as_db_model( + b.id), + b.build_node_stats.as_db_model(b.id)) except Exception as err: # pylint: disable=broad-except logging.error( - 'failed to update build task %d: %s', - b.id, err, exc_info=True) + 'build: %d, failed to update build task %d: %s', + b.build_id, b.id, err, exc_info=True) else: - logging.info('build task %d was updated', b.id) + logging.info( + 'build: %d, build task %d was updated', b.build_id, b.id) else: logging.info( - "build_task %d is still %s. Skipping", b.id, BuildTaskEnum(b.status_id).name) + "build: %d, build_task %d is still %s. Skipping", + b.build_id, b.id, BuildTaskEnum(b.status_id).name) def update_builds(self): logging.info('Getting list of tasks from DB') @@ -80,7 +101,7 @@ class Extractor: logging.info('Updating build tasks') build_tasks_to_check = [ b for b in build.build_tasks if b.id in build_tasks_db] - self.__update_build_tasks_statuses( + self.__update_build_tasks( build_tasks_to_check, build_tasks_db) if build.finished_at: diff --git a/build_analitycs/build_analytics/extractor/start.py b/build_analytics/build_analytics/extractor/start.py similarity index 100% rename from build_analitycs/build_analytics/extractor/start.py rename to build_analytics/build_analytics/extractor/start.py diff --git a/build_analitycs/build_analytics/models/__init__.py b/build_analytics/build_analytics/models/__init__.py similarity index 100% rename from build_analitycs/build_analytics/models/__init__.py rename to build_analytics/build_analytics/models/__init__.py diff --git a/build_analitycs/build_analytics/models/build.py b/build_analytics/build_analytics/models/build.py similarity index 100% rename from build_analitycs/build_analytics/models/build.py rename to build_analytics/build_analytics/models/build.py diff --git a/build_analitycs/build_analytics/models/build_db.py b/build_analytics/build_analytics/models/build_db.py similarity index 100% rename from build_analitycs/build_analytics/models/build_db.py rename to build_analytics/build_analytics/models/build_db.py diff --git a/build_analytics/build_analytics/models/build_node_stat_db.py b/build_analytics/build_analytics/models/build_node_stat_db.py new file mode 100644 index 0000000..5e0fb5a --- /dev/null +++ b/build_analytics/build_analytics/models/build_node_stat_db.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel # pylint: disable=no-name-in-module +from typing import Optional + + +class BuildNodeStatDB(BaseModel): + """ + Build node stat as it sent to/received from database + """ + build_task_id: int + stat_name_id: int + start_ts: Optional[float] = None + end_ts: Optional[float] = None diff --git a/build_analytics/build_analytics/models/build_node_stats.py b/build_analytics/build_analytics/models/build_node_stats.py new file mode 100644 index 0000000..a71715d --- /dev/null +++ b/build_analytics/build_analytics/models/build_node_stats.py @@ -0,0 +1,41 @@ +from typing import List + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +from .build_stat import BuildStat +from .build_node_stat_db import BuildNodeStatDB +from .enums import BuildNodeStatsEnum + + +class BuildNodeStats(BaseModel): + """ + Represents build statistics for build node + """ + build_all: BuildStat + build_binaries: BuildStat + build_packages: BuildStat + build_srpm: BuildStat + build_node_task: BuildStat + cas_notarize_artifacts: BuildStat + cas_source_authenticate: BuildStat + git_checkout: BuildStat + upload: BuildStat + + def as_db_model(self, build_task_id: int) -> List[BuildNodeStatDB]: + result = [] + for field_name in self.__fields__.keys(): + + stats: BuildStat = getattr(self, field_name) + start_ts = stats.start_ts.timestamp() \ + if stats.start_ts else None + end_ts = stats.end_ts.timestamp() \ + if stats.end_ts else None + stat_name_id = BuildNodeStatsEnum[field_name].value + + build_node_stat_db = BuildNodeStatDB(build_task_id=build_task_id, + stat_name_id=stat_name_id, + start_ts=start_ts, + end_ts=end_ts) + result.append(build_node_stat_db) + return result diff --git a/build_analytics/build_analytics/models/build_stat.py b/build_analytics/build_analytics/models/build_stat.py new file mode 100644 index 0000000..ab1fa7f --- /dev/null +++ b/build_analytics/build_analytics/models/build_stat.py @@ -0,0 +1,15 @@ +""" +Module for BuildStat model +""" + +from datetime import datetime +from typing import Optional +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +class BuildStat(BaseModel): + """ + BuildStat represents particular build statistic + """ + start_ts: Optional[datetime] = None + end_ts: Optional[datetime] = None diff --git a/build_analytics/build_analytics/models/build_stat_db.py b/build_analytics/build_analytics/models/build_stat_db.py new file mode 100644 index 0000000..5b7240f --- /dev/null +++ b/build_analytics/build_analytics/models/build_stat_db.py @@ -0,0 +1,16 @@ +""" +Module for BuildStatDB model +""" + + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +class BuildStatDB(BaseModel): + """ + Represents build stat as it send to/received from database + """ + build_task_id: int + stat_name_id: int + start_ts: float + end_ts: float diff --git a/build_analitycs/build_analytics/models/build_task.py b/build_analytics/build_analytics/models/build_task.py similarity index 82% rename from build_analitycs/build_analytics/models/build_task.py rename to build_analytics/build_analytics/models/build_task.py index d4016d4..c84378e 100644 --- a/build_analitycs/build_analytics/models/build_task.py +++ b/build_analytics/build_analytics/models/build_task.py @@ -1,10 +1,12 @@ from datetime import datetime -from typing import Optional +from typing import Optional, Tuple from pydantic import BaseModel # pylint: disable=no-name-in-module from .build_task_db import BuildTaskDB +from .build_node_stats import BuildNodeStats from .enums import ArchEnum +from .web_node_stats import WebNodeStats class BuildTask(BaseModel): @@ -15,6 +17,8 @@ class BuildTask(BaseModel): started_at: Optional[datetime] = None finished_at: Optional[datetime] = None status_id: int + build_node_stats: BuildNodeStats + web_node_stats: WebNodeStats def as_db_model(self) -> BuildTaskDB: started_at = self.started_at.timestamp() \ diff --git a/build_analitycs/build_analytics/models/build_task_db.py b/build_analytics/build_analytics/models/build_task_db.py similarity index 100% rename from build_analitycs/build_analytics/models/build_task_db.py rename to build_analytics/build_analytics/models/build_task_db.py diff --git a/build_analitycs/build_analytics/models/db_config.py b/build_analytics/build_analytics/models/db_config.py similarity index 100% rename from build_analitycs/build_analytics/models/db_config.py rename to build_analytics/build_analytics/models/db_config.py diff --git a/build_analytics/build_analytics/models/enums.py b/build_analytics/build_analytics/models/enums.py new file mode 100644 index 0000000..0b63a09 --- /dev/null +++ b/build_analytics/build_analytics/models/enums.py @@ -0,0 +1,37 @@ +# pylint: disable=invalid-name + +from enum import IntEnum + + +class ArchEnum(IntEnum): + i686 = 0 + x86_64 = 1 + aarch64 = 2 + ppc64le = 3 + s390x = 4 + + +class BuildTaskEnum(IntEnum): + idle = 0 + started = 1 + completed = 2 + failed = 3 + excluded = 4 + + +class WebNodeStatsEnum(IntEnum): + build_done = 0 + logs_processing = 1 + packages_processing = 2 + + +class BuildNodeStatsEnum(IntEnum): + upload = 0 + build_all = 1 + build_srpm = 2 + git_checkout = 3 + build_binaries = 4 + build_packages = 5 + build_node_task = 6 + cas_notarize_artifacts = 7 + cas_source_authenticate = 8 diff --git a/build_analitycs/build_analytics/models/extractor_config.py b/build_analytics/build_analytics/models/extractor_config.py similarity index 82% rename from build_analitycs/build_analytics/models/extractor_config.py rename to build_analytics/build_analytics/models/extractor_config.py index 8b8f58a..a49c3dd 100644 --- a/build_analitycs/build_analytics/models/extractor_config.py +++ b/build_analytics/build_analytics/models/extractor_config.py @@ -10,6 +10,7 @@ ALBS_URL_DEFAULT = 'https://build.almalinux.org' LOG_FILE_DEFAULT = '/tmp/extractor.log' API_DEFAULT = 30 SCRAPE_INTERVAL_DEFAULT = 3600 +START_FROM_DEFAULT = 5808 class ExtractorConfig(BaseModel): @@ -21,7 +22,7 @@ class ExtractorConfig(BaseModel): albs_url: HttpUrl = Field(description='ALBS root URL', default=ALBS_URL_DEFAULT) oldest_build_age: datetime = \ - Field(description='oldest build age to extract and store') + Field(description='oldest build age to store') jwt: str = Field(description='ALBS JWT token') db_config: DbConfig = Field(description="database configuration") api_timeout: int = Field( @@ -29,3 +30,5 @@ class ExtractorConfig(BaseModel): default=API_DEFAULT) scrape_interval: int = Field(description='how often (in seconds) we will extract data from ALBS', default=SCRAPE_INTERVAL_DEFAULT) + start_from: int = Field(description='build id to start populating empty db with', + default=START_FROM_DEFAULT) diff --git a/build_analitycs/build_analytics/models/sign_task_db.py b/build_analytics/build_analytics/models/sign_task_db.py similarity index 100% rename from build_analitycs/build_analytics/models/sign_task_db.py rename to build_analytics/build_analytics/models/sign_task_db.py diff --git a/build_analytics/build_analytics/models/web_node_stat_db.py b/build_analytics/build_analytics/models/web_node_stat_db.py new file mode 100644 index 0000000..dd28225 --- /dev/null +++ b/build_analytics/build_analytics/models/web_node_stat_db.py @@ -0,0 +1,13 @@ + +from pydantic import BaseModel # pylint: disable=no-name-in-module +from typing import Optional + + +class WebNodeStatDB(BaseModel): + """ + Represents WebNodeStat as it sent to/received from databse + """ + build_task_id: int + stat_name_id: int + start_ts: Optional[float] = None + end_ts: Optional[float] = None diff --git a/build_analytics/build_analytics/models/web_node_stats.py b/build_analytics/build_analytics/models/web_node_stats.py new file mode 100644 index 0000000..46f77c6 --- /dev/null +++ b/build_analytics/build_analytics/models/web_node_stats.py @@ -0,0 +1,35 @@ +from typing import List + +from pydantic import BaseModel # pylint: disable=no-name-in-module + + +from .build_stat import BuildStat +from .web_node_stat_db import WebNodeStatDB +from .enums import WebNodeStatsEnum + + +class WebNodeStats(BaseModel): + """ + Represents build statistics for web node + """ + build_done: BuildStat + logs_processing: BuildStat + packages_processing: BuildStat + + def as_db_model(self, build_task_id: int) -> List[WebNodeStatDB]: + result = [] + for field_name in self.__fields__.keys(): + + stats: BuildStat = getattr(self, field_name) + start_ts = stats.start_ts.timestamp() \ + if stats.start_ts else None + end_ts = stats.end_ts.timestamp() \ + if stats.end_ts else None + stat_name_id = WebNodeStatsEnum[field_name].value + + web_node_stat_db = WebNodeStatDB(build_task_id=build_task_id, + stat_name_id=stat_name_id, + start_ts=start_ts, + end_ts=end_ts) + result.append(web_node_stat_db) + return result diff --git a/build_analitycs/config_default.yml b/build_analytics/config_default.yml similarity index 87% rename from build_analitycs/config_default.yml rename to build_analytics/config_default.yml index 70349bc..c7b5e66 100644 --- a/build_analitycs/config_default.yml +++ b/build_analytics/config_default.yml @@ -55,4 +55,9 @@ data_store_days: 30 # sleep time in seconds between data extraction # required: no # default: 3600 -scrape_interval: 3600 \ No newline at end of file +scrape_interval: 3600 + +# build_id to start populating empty db with +# required: false +# default: 5808 (first build with correct metrics) +start_from: 5808 diff --git a/build_analitycs/db_schema/postgres.sql b/build_analytics/migrations/1.sql similarity index 51% rename from build_analitycs/db_schema/postgres.sql rename to build_analytics/migrations/1.sql index 036e972..40c6029 100644 --- a/build_analitycs/db_schema/postgres.sql +++ b/build_analytics/migrations/1.sql @@ -1,5 +1,6 @@ +BEGIN; + -- builds -DROP TABLE IF EXISTS builds CASCADE; CREATE TABLE builds ( id INTEGER PRIMARY KEY, url VARCHAR(50) NOT NULL, @@ -16,7 +17,6 @@ ON builds(finished_at); -- build_taks_enum -DROP TABLE IF EXISTS build_task_status_enum CASCADE; CREATE TABLE IF NOT EXISTS build_task_status_enum( id INTEGER PRIMARY KEY, value VARCHAR(15) @@ -32,7 +32,6 @@ VALUES -- arch_enum -DROP TABLE IF EXISTS arch_enum CASCADE; CREATE TABLE arch_enum( id INTEGER PRIMARY KEY, value VARCHAR(15) @@ -47,8 +46,39 @@ VALUES (4, 's390x'); +-- web_node_stats_enum +CREATE TABLE web_node_stats_enum ( + id INTEGER PRIMARY KEY, + value VARCHAR(50) +); + +INSERT INTO web_node_stats_enum (id, value) +VALUEs + (0, 'build_done'), + (1, 'logs_processing'), + (2, 'packages_processing'); + + +-- build_node_stats_enum +CREATE TABLE build_node_stats_enum( + id INTEGER PRIMARY KEY, + value VARCHAR(50) +); + +INSERT INTO build_node_stats_enum (id, value) +VALUES + (0, 'upload'), + (1, 'build_all'), + (2, 'build_srpm'), + (3, 'git_checkout'), + (4, 'build_binaries'), + (5, 'build_packages'), + (6, 'build_node_task'), + (7, 'cas_notarize_artifacts'), + (8, 'cas_source_authenticate'); + + -- build_tasks -DROP TABLE IF EXISTS build_tasks CASCADE; CREATE TABLE build_tasks ( id INTEGER PRIMARY KEY, name VARCHAR(50) NOT NULL, @@ -69,8 +99,43 @@ CREATE INDEX build_tasks_finished_at ON build_tasks(finished_at); +-- web_node_stats +CREATE TABLE web_node_stats ( + build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, + stat_name_id INTEGER REFERENCES web_node_stats_enum(id) ON DELETE SET NULL, + start_ts REAL, + end_ts REAL +); + +CREATE INDEX web_node_stats_build_task_id +ON web_node_stats(build_task_id); + +CREATE INDEX web_node_stats_start_ts +ON web_node_stats(start_ts); + +CREATE INDEX web_node_stats_end_ts +ON web_node_stats(end_ts); + + +-- build_node_stats +CREATE TABLE build_node_stats ( + build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE, + stat_name_id INTEGER REFERENCES build_node_stats_enum(id) ON DELETE SET NULL, + start_ts REAL, + end_ts REAL +); + +CREATE INDEX build_node_stats_build_task_id +ON build_node_stats(build_task_id); + +CREATE INDEX build_node_stats_build_start_ts +ON build_node_stats(start_ts); + +CREATE INDEX build_node_stats_build_end_ts +ON build_node_stats(end_ts); + + -- sign_tasks -DROP TABLE IF EXISTS sign_tasks CASCADE; CREATE TABLE sign_tasks ( id INTEGER PRIMARY KEY, build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE, @@ -90,3 +155,14 @@ ON sign_tasks(started_at); CREATE INDEX sign_tasks_finished_at ON sign_tasks(finished_at); + + + +-- schema_version +CREATE TABLE schema_version ( + version INTEGER +); +INSERT INTO schema_version (version) +VALUES (1); + +COMMIT; \ No newline at end of file diff --git a/build_analytics/releases.txt b/build_analytics/releases.txt new file mode 100644 index 0000000..409ef5f --- /dev/null +++ b/build_analytics/releases.txt @@ -0,0 +1,5 @@ +0.1.0 (2023-03-01) +First version + +0.2.0 +New parameter start_from \ No newline at end of file diff --git a/build_analitycs/requirements.txt b/build_analytics/requirements.txt similarity index 100% rename from build_analitycs/requirements.txt rename to build_analytics/requirements.txt diff --git a/build_analitycs/run_extractor.py b/build_analytics/run_extractor.py similarity index 100% rename from build_analitycs/run_extractor.py rename to build_analytics/run_extractor.py diff --git a/grafana-dashbords/Build analytics.json b/grafana-dashbords/Build analytics.json new file mode 100644 index 0000000..c0748d8 --- /dev/null +++ b/grafana-dashbords/Build analytics.json @@ -0,0 +1,1509 @@ +{ + "__inputs": [ + { + "name": "DS_ALBS_ANALYTICS", + "label": "albs_analytics", + "description": "", + "type": "datasource", + "pluginId": "postgres", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.6" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "postgres", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "finished" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS finished\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at is NOT NULL;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS running\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at is NULL;", + "refId": "C", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Builds", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS idle FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 0;", + "refId": "idle", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS started FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 1;", + "refId": "started", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS completed FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 2;", + "refId": "completed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS failed FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 3;", + "refId": "failed", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS excluded FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND status_id = 4;", + "refId": "excluded", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by status", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "excluded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "x86_64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "i686" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "aarch64" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ppc64le" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS i686 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 0;", + "refId": "i686", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS x86_64 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 1;", + "refId": "x86_64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS aarch64 FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 2;", + "refId": "aarch64", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS ppc64le FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 3;", + "refId": "ppc64le", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT COUNT(id) AS s390x FROM build_tasks \nWHERE $__unixEpochFilter(started_at) AND arch_id = 4;", + "refId": "excluded", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Build tasks by arch", + "type": "piechart" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MAX(finished_at - created_at) AS MAX\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "MAX", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MIN(finished_at - created_at) AS MIN\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "MIN", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT AVG(finished_at - created_at) AS AVG\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL;", + "refId": "AVG", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build duration", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MAX(finished_at - started_at) AS MAX\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "MAX", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT MIN(finished_at - started_at) AS MIN\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "MIN", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + }, + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT AVG(finished_at - started_at) AS AVG\nFROM build_tasks\nWHERE $__unixEpochFilter(started_at) AND finished_at IS NOT NULL AND status_id <> 4", + "refId": "AVG", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Build task duration", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 5, + "panels": [], + "title": "Build details", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 287 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.width", + "value": 107 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "hide": false, + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT \n b.id, \n b.url, \n b.created_at * 1000 as created,\n b.finished_at * 1000 as finished,\n (SELECT COUNT(id) FROM build_tasks AS bt WHERE bt.build_id = b.id) AS task_count,\n b.finished_at - b.created_at as duration\nFROM builds as b \nWHERE $__unixEpochFilter(created_at) AND b.finished_at IS NOT NULL", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished builds", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "created" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 7, + "panels": [], + "title": "Build tasks details", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "duration" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 15, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "duration" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "cacheDurationSeconds": 300, + "datasource": { + "type": "postgres", + "uid": "${DS_ALBS_ANALYTICS}" + }, + "editorMode": "code", + "fields": [ + { + "jsonPath": "" + } + ], + "format": "table", + "method": "GET", + "queryParams": "", + "rawQuery": true, + "rawSql": "SELECT\n bt.id,\n bt.name,\n bt.build_id,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE $__unixEpochFilter(started_at) AND status_id >= 2;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "urlPath": "" + } + ], + "title": "Finished buld tasks", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "started" + }, + { + "destinationType": "time", + "targetField": "finished" + } + ], + "fields": {} + } + } + ], + "type": "table" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Build analytics", + "uid": "02mg4oxVk", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana-dashbords/albs_analytics.json b/grafana-dashbords/albs_analytics.json deleted file mode 100644 index 6aa83f8..0000000 --- a/grafana-dashbords/albs_analytics.json +++ /dev/null @@ -1,280 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_POSTGRESQL", - "label": "PostgreSQL", - "description": "", - "type": "datasource", - "pluginId": "postgres", - "pluginName": "PostgreSQL" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "9.3.2" - }, - { - "type": "datasource", - "id": "postgres", - "name": "PostgreSQL", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto", - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "id" - }, - "properties": [ - { - "id": "custom.width", - "value": 54 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "created_at" - }, - "properties": [ - { - "id": "custom.width", - "value": 226 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "finished_at" - }, - "properties": [ - { - "id": "custom.width", - "value": 209 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "finished" - }, - "properties": [ - { - "id": "custom.width", - "value": 187 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "created" - }, - "properties": [ - { - "id": "custom.width", - "value": 213 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "url" - }, - "properties": [ - { - "id": "custom.width", - "value": 279 - } - ] - } - ] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "footer": { - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "duration (h)" - } - ] - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "cacheDurationSeconds": 300, - "datasource": { - "type": "postgres", - "uid": "${DS_POSTGRESQL}" - }, - "editorMode": "code", - "fields": [ - { - "jsonPath": "" - } - ], - "format": "table", - "hide": false, - "method": "GET", - "queryParams": "", - "rawQuery": true, - "rawSql": "SELECT id, url, created_at * 1000 as created, finished_at * 1000 as finished, (finished_at - created_at) / (60*60) as duration\nFROM builds\nWHERE $__unixEpochFilter(created_at) AND finished_at IS NOT NULL", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "urlPath": "" - } - ], - "title": "Finished builds", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "created" - }, - { - "destinationType": "time", - "targetField": "finished" - } - ], - "fields": {} - } - }, - { - "id": "organize", - "options": { - "excludeByName": {}, - "indexByName": {}, - "renameByName": { - "duration": "duration (h)" - } - } - } - ], - "type": "table" - } - ], - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-3h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "albs_analytics", - "uid": "02mg4oxVk", - "version": 1, - "weekStart": "" -} \ No newline at end of file