kzhukov
/
albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

ALBS-1043: 

- db,extractor: added check for cases when build 'moved' between pages
 - grafana: new dashboard Test tasks.json, added test info to other dashboars
This commit is contained in:
Kirill Zhukov 2023-03-22 11:30:12 +01:00
parent 4d5ffcc74f
commit 5b1e296fbc
6 changed files with 5408 additions and 2624 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build_analytics/build_analytics/db.py
View File

@ -26,6 +26,17 @@ class DB():
def __del__(self):
self.close_conn()
def build_exists(self, build_id: int) -> bool:
sql = f'''
SELECT COUNT(id)
FROM builds
WHERE id = %s;
'''
cur = self.__conn.cursor()
cur.execute(sql, (build_id,))
val = int(cur.fetchone()[0])
return val == 1
def insert_build(self, build: BuildDB):
sql = '''
INSERT INTO builds(id, url, created_at, finished_at)

4
build_analytics/build_analytics/extractor/extractor.py
View File

@ -34,6 +34,10 @@ class Extractor:
stop = True
break
# some builds could move from one page to another
if self.db.build_exists(build_id=build.id):
continue
# inserting build build tasks and build tasks statistics
logging.info('inserting %s', build.id)
try:

894
build_analytics/grafana-dashbords/Build analytics.json
View File

@ -87,7 +87,7 @@
},
"id": 17,
"panels": [],
"title": "Summary",
"title": "Builds",
"type": "row"
},
{
@ -1427,7 +1427,7 @@
"type": "stat"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@ -1435,7 +1435,10 @@
"y": 13
},
"id": 26,
"panels": [
"panels": [],
"title": "Build Steps",
"type": "row"
},
{
"datasource": {
"type": "postgres",
@ -1879,7 +1882,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
},
@ -3267,10 +3271,487 @@
],
"title": "upload",
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 29
},
"id": 49,
"panels": [],
"title": "Tests",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 13,
"x": 0,
"y": 30
},
"id": 45,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n COUNT(tt.id),\n enum.value AS status\nFROM test_tasks AS tt\nINNER JOIN test_tasks_status_enum AS enum\nON tt.status_id = enum.id\nWHERE $__unixEpochFilter(started_at)\nGROUP BY (status);\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"title": "Build Steps",
"type": "row"
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests count (group by status)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 11,
"x": 13,
"y": 30
},
"id": 56,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n MAX(t.duration) AS \"MAX\",\n MIN(t.duration) AS \"MIN\",\n AVG(t.duration) AS \"AVERAGE\",\n PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY t.duration) as \"MEDIAN\",\n PERCENTILE_CONT(0.95) WITHIN GROUP(ORDER BY t.duration) as \"95TH PERCENTILE\"\nFROM\n (SELECT \n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\n FROM test_tasks AS tt\n INNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\n INNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\n INNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE $__unixEpochFilter(tt.started_at)\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\n WHERE $__unixEpochFilter(tt.started_at)\n ) AS t;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Tests duration",
"type": "stat"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 36
},
"id": 58,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE $__unixEpochFilter(tt.started_at)\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 42
},
"id": 59,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n aenum.value AS arch,\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN build_tasks as bt\n ON tt.build_task_id = bt.id\nINNER JOIN arch_enum as aenum\n ON bt.arch_id = aenum.id\nWHERE $__unixEpochFilter(tt.started_at)\nGROUP BY (aenum.value); \n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by arch)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 16,
"x": 8,
"y": 42
},
"id": 47,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nWHERE $__unixEpochFilter(started_at)\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"collapsed": true,
@ -3278,7 +3759,7 @@
"h": 1,
"w": 24,
"x": 0,
"y": 14
"y": 49
},
"id": 24,
"panels": [
@ -3304,8 +3785,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3374,7 +3854,7 @@
"h": 15,
"w": 24,
"x": 0,
"y": 15
"y": 62
},
"id": 2,
"options": {
@ -3456,22 +3936,19 @@
"type": "table"
}
],
"title": "Builds",
"title": "Details: builds",
"type": "row"
},
{
"collapsed": false,
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 15
"y": 50
},
"id": 39,
"panels": [],
"title": "Build tasks",
"type": "row"
},
"panels": [
{
"datasource": {
"type": "postgres",
@ -3494,8 +3971,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3653,7 +4129,7 @@
"h": 9,
"w": 24,
"x": 0,
"y": 16
"y": 61
},
"id": 37,
"options": {
@ -3791,8 +4267,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -3898,7 +4373,7 @@
"h": 10,
"w": 24,
"x": 0,
"y": 25
"y": 70
},
"id": 43,
"options": {
@ -3979,6 +4454,377 @@
"type": "table"
}
],
"title": "Details: build tasks",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 51
},
"id": 53,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 322
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 93
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 62
}
]
},
{
"matcher": {
"id": "byName",
"options": "build task id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}"
}
]
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 62
},
"id": 54,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.build_task_id as \"build task id\",\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE $__unixEpochFilter(tt.started_at)\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE $__unixEpochFilter(tt.started_at);\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 69
},
"id": 61,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tss.test_task_id AS \"test task id\", \n tt.package_fullname AS \"package name\",\n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE $__unixEpochFilter(tt.started_at);\n",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: tests",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 37,
"style": "dark",
@ -3994,6 +4840,6 @@
"timezone": "",
"title": "Build analytics",
"uid": "02mg4oxVk",
"version": 64,
"version": 84,
"weekStart": ""
}

989
build_analytics/grafana-dashbords/Build details.json
View File

File diff suppressed because it is too large Load Diff

602
build_analytics/grafana-dashbords/Build task details.json
View File

@ -35,6 +35,12 @@
"name": "Grafana",
"version": "9.3.2"
},
{
"type": "panel",
"id": "piechart",
"name": "Pie chart",
"version": ""
},
{
"type": "datasource",
"id": "postgres",
@ -77,6 +83,17 @@
"links": [],
"liveNow": false,
"panels": [
{
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 28,
"title": "Build task",
"type": "row"
},
{
"datasource": {
"type": "postgres",
@ -143,7 +160,7 @@
"h": 3,
"w": 24,
"x": 0,
"y": 0
"y": 1
},
"id": 5,
"options": {
@ -196,7 +213,7 @@
"urlPath": ""
}
],
"title": "Task summary",
"title": "Build task",
"transformations": [
{
"id": "convertFieldType",
@ -259,10 +276,10 @@
]
},
"gridPos": {
"h": 9,
"h": 8,
"w": 24,
"x": 0,
"y": 3
"y": 4
},
"id": 12,
"options": {
@ -629,14 +646,219 @@
"type": "bargauge"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 12
},
"id": 8,
"id": 22,
"panels": [],
"title": "Test tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 11,
"x": 0,
"y": 13
},
"id": 24,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 13,
"x": 11,
"y": 13
},
"id": 18,
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"showLegend": true,
"values": [
"value"
]
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nWHERE tt.build_task_id = $build_task\nGROUP BY (tt.package_fullname); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
}
],
"title": "Tests duration (group by package)",
"transformations": [
{
"id": "rowsToFields",
"options": {
"mappings": []
}
}
],
"type": "piechart"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 21
},
"id": 20,
"panels": [
{
"datasource": {
@ -650,7 +872,7 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
@ -712,7 +934,7 @@
"h": 5,
"w": 24,
"x": 0,
"y": 13
"y": 22
},
"id": 2,
"options": {
@ -750,7 +972,7 @@
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"rawSql": "SELECT \n enum.value as step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM web_node_stats AS stats\nINNER JOIN web_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
@ -805,7 +1027,7 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
@ -852,10 +1074,10 @@
]
},
"gridPos": {
"h": 11,
"h": 8,
"w": 24,
"x": 0,
"y": 18
"y": 27
},
"id": 3,
"options": {
@ -893,7 +1115,7 @@
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"rawSql": "SELECT enum.value AS step,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
@ -937,7 +1159,357 @@
"type": "table"
}
],
"title": "Details",
"title": "Details: build task",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 22
},
"id": 30,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 389
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 72
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "revision"
},
"properties": [
{
"id": "custom.width",
"value": 119
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 23
},
"id": 26,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "id"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.build_task_id = $build_task\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 28
},
"id": 16,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.build_task_id = $build_task; ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"title": "Details: test tasks",
"type": "row"
}
],
@ -975,6 +1547,6 @@
"timezone": "",
"title": "Build task details",
"uid": "vtQClqxVk",
"version": 37,
"version": 61,
"weekStart": ""
}

516
build_analytics/grafana-dashbords/Test task details.json Normal file
View File

@ -0,0 +1,516 @@
{
"__inputs": [
{
"name": "DS_POSTGRESQL",
"label": "PostgreSQL",
"description": "",
"type": "datasource",
"pluginId": "postgres",
"pluginName": "PostgreSQL"
}
],
"__elements": {},
"__requires": [
{
"type": "panel",
"id": "bargauge",
"name": "Bar gauge",
"version": ""
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.3.2"
},
{
"type": "datasource",
"id": "postgres",
"name": "PostgreSQL",
"version": "1.0.0"
},
{
"type": "panel",
"id": "table",
"name": "Table",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": false,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package"
},
"properties": [
{
"id": "custom.width",
"value": 253
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 80
}
]
}
]
},
"gridPos": {
"h": 3,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": []
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n DISTINCT tt.id,\n tt.package_fullname AS package,\n tt.revision,\n enum.value AS \"status\",\n tt.started_at * 1000 AS \"started at\",\n tf.finished_at * 1000 AS \"finished at\", \n tf.finished_at - tt.started_at AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_tasks_status_enum AS enum\n ON tt.status_id = enum.id\nINNER JOIN \n (SELECT \n tss.test_task_id, \n MAX(tss.finish_ts) AS finished_at \n FROM test_steps_stats AS tss\n INNER JOIN test_tasks AS tt\n ON tss.test_task_id = tt.id\n WHERE tt.id = $id\n GROUP BY tss.test_task_id) AS tf\n ON tf.test_task_id = tt.id\nWHERE tt.id = $id;",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Task info",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started at"
},
{
"destinationType": "time",
"targetField": "finished at"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 3
},
"id": 8,
"options": {
"displayMode": "basic",
"minVizHeight": 10,
"minVizWidth": 0,
"orientation": "vertical",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n enum.value AS \"step name\",\n SUM(tss.finish_ts - tss.start_ts) AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id\nGROUP BY (enum.value); ",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test duration (group by test step)",
"transformations": [
{
"id": "rowsToFields",
"options": {}
}
],
"type": "bargauge"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "test task id"
},
"properties": [
{
"id": "custom.width",
"value": 159
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/8nFXlkB4z/test-task-details?orgId=1&var-id=${__value.raw}"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "package name"
},
"properties": [
{
"id": "custom.width",
"value": 408
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 11
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT \n tt.package_fullname AS \"package name\",\n tss.test_task_id AS \"test task id\", \n enum.value AS \"step name\",\n tss.start_ts * 1000 AS started,\n tss.finish_ts * 1000 AS finished,\n tss.finish_ts - tss.start_ts AS duration\nFROM test_tasks AS tt\nINNER JOIN test_steps_stats AS tss\n ON tt.id = tss.test_task_id\nINNER JOIN test_steps_enum AS enum\n ON tss.stat_name_id = enum.id\nWHERE tt.id = $id",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Test steps",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {},
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"definition": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"hide": 0,
"includeAll": false,
"label": "Test task id",
"multi": false,
"name": "id",
"options": [],
"query": "SELECT id\nFROM test_tasks\nORDER BY id DESC",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Test task details",
"uid": "8nFXlkB4z",
"version": 8,
"weekStart": ""
}