kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

ALBS-1026

Browse Source 
db.py: bug fix with updates
grafana: added build task steps
This commit is contained in:
Kirill Zhukov 2023-03-09 13:33:02 +01:00
parent d1291c7308
commit fafd7ae79b
4 changed files with 2812 additions and 774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build_analytics/build_analytics/db.py
View File

@ -151,9 +151,10 @@ class DB():
UPDATE web_node_stats
SET start_ts = %s,
end_ts = %s
WHERE build_task_id = %s;
WHERE build_task_id = %s AND stat_name_id = %s;
'''
cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id))
cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
# updating build_node_stats
for stat in build_node_stats:
@ -161,9 +162,10 @@ class DB():
UPDATE build_node_stats
SET start_ts = %s,
end_ts = %s
WHERE build_task_id = %s;
WHERE build_task_id = %s and stat_name_id = %s;
'''
cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id))
cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
# commiting changes
self.__conn.commit()
@ -177,7 +179,6 @@ class DB():
cur = self.__conn.cursor()
cur.execute(sql)
val = cur.fetchone()
print(val)
if not val:
return None
return int(val[0])

3003
grafana-dashbords/Build analytics.json
View File

File diff suppressed because it is too large Load Diff

6
grafana-dashbords/Build details.json
View File

@ -161,7 +161,7 @@
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id AND status_id >= 2;",
"rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id;",
"refId": "A",
"sql": {
"columns": [
@ -183,7 +183,7 @@
"urlPath": ""
}
],
"title": "Finished buld tasks",
"title": "Build tasks",
"transformations": [
{
"id": "convertFieldType",
@ -239,6 +239,6 @@
"timezone": "",
"title": "Build details",
"uid": "dmVtrz-4k",
"version": 5,
"version": 8,
"weekStart": ""
}

566
grafana-dashbords/Build task details.json
View File

@ -41,12 +41,6 @@
"name": "PostgreSQL",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "table",
@ -229,7 +223,23 @@
},
"unit": "s"
},
"overrides": []
"overrides": [
{
"matcher": {
"id": "byName",
"options": "wait time"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 9,
@ -254,49 +264,6 @@
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "upload",
"sql": {
"columns": [
{
"parameters": [
{
"name": "start_ts",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "build_node_stats",
"urlPath": ""
},
{
"datasource": {
"type": "postgres",
@ -306,8 +273,8 @@
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2",
"refId": "build_srpm",
"rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3",
"refId": "git_checkout",
"sql": {
"columns": [
{
@ -335,8 +302,37 @@
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3",
"refId": "git_checkout",
"rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;",
"refId": "cas_source_authenticate",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2",
"refId": "build_srpm",
"sql": {
"columns": [
{
@ -413,35 +409,6 @@
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;",
"refId": "cas_source_authenticate",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
@ -499,6 +466,146 @@
],
"limit": 50
}
},
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "upload",
"sql": {
"columns": [
{
"parameters": [
{
"name": "start_ts",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "build_node_stats",
"urlPath": ""
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_node_task\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 6",
"refId": "build_node_task",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as web_node_task\nFROM web_node_stats\nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "web_node_task",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT\n finished_at - started_at AS duration \nFROM build_tasks \nWHERE id = $build_task;",
"refId": "task_diration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "__expr__",
"uid": "${DS_EXPRESSION}"
},
"expression": "$task_diration - $build_node_task - $web_node_task",
"hide": false,
"refId": "wait time",
"type": "math"
}
],
"title": "Task duration",
@ -605,7 +712,12 @@
},
"frameIndex": 1,
"showHeader": true,
"sortBy": []
"sortBy": [
{
"desc": true,
"displayName": "started"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
@ -690,8 +802,7 @@
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
"color": "green"
},
{
"color": "red",
@ -811,287 +922,6 @@
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"filterable": true,
"inspect": true
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 29
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Build node stats",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 17,
"x": 3,
"y": 40
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT\n bt.finished_at - bt.started_at AS task_duration \nFROM build_tasks AS bt\nWHERE bt.id = $build_task;",
"refId": "task_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n stats.end_ts - stats.start_ts AS web_node_duration\nFROM web_node_stats AS stats\nWHERE build_task_id = $build_task and stat_name_id=0;",
"refId": "web_node_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n stats.end_ts - stats.start_ts AS build_node_duration\nFROM build_node_stats AS stats\nWHERE build_task_id = $build_task AND stat_name_id=6",
"refId": "build_node_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "__expr__",
"uid": "${DS_EXPRESSION}"
},
"expression": "$task_duration - ($web_node_duration + $build_node_duration)",
"hide": false,
"refId": "Not accounted duration",
"type": "math"
}
],
"title": "Diff",
"type": "stat"
}
],
"schemaVersion": 37,
@ -1128,6 +958,6 @@
"timezone": "",
"title": "Build task details",
"uid": "vtQClqxVk",
"version": 27,
"version": 34,
"weekStart": ""
}