kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

ALBS-1026

Browse Source 
db.py: bug fix with updates
grafana: added build task steps
This commit is contained in:
Kirill Zhukov 2023-03-09 13:33:02 +01:00
parent d1291c7308
commit fafd7ae79b
4 changed files with 2812 additions and 774 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build_analytics/build_analytics/db.py
View File

@ -151,9 +151,10 @@ class DB():
UPDATE web_node_stats UPDATE web_node_stats
SET start_ts = %s, SET start_ts = %s,
end_ts = %s end_ts = %s
WHERE build_task_id = %s; WHERE build_task_id = %s AND stat_name_id = %s;
''' '''
cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
# updating build_node_stats # updating build_node_stats
for stat in build_node_stats: for stat in build_node_stats:
@ -161,9 +162,10 @@ class DB():
UPDATE build_node_stats UPDATE build_node_stats
SET start_ts = %s, SET start_ts = %s,
end_ts = %s end_ts = %s
WHERE build_task_id = %s; WHERE build_task_id = %s and stat_name_id = %s;
''' '''
cur.execute(sql, (stat.start_ts, stat.end_ts, build_task.id)) cur.execute(sql, (stat.start_ts, stat.end_ts,
build_task.id, stat.stat_name_id))
# commiting changes # commiting changes
self.__conn.commit() self.__conn.commit()
@ -177,7 +179,6 @@ class DB():
cur = self.__conn.cursor() cur = self.__conn.cursor()
cur.execute(sql) cur.execute(sql)
val = cur.fetchone() val = cur.fetchone()
print(val)
if not val: if not val:
return None return None
return int(val[0]) return int(val[0])

3003
grafana-dashbords/Build analytics.json
View File

File diff suppressed because it is too large Load Diff

6
grafana-dashbords/Build details.json
View File

@ -161,7 +161,7 @@
"method": "GET", "method": "GET",
"queryParams": "", "queryParams": "",
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id AND status_id >= 2;", "rawSql": "SELECT\n bt.id,\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS duration \nFROM build_tasks AS bt\nJOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\nJOIN arch_enum on bt.arch_id = arch_enum.id\nWHERE bt.build_id = $build_id;",
"refId": "A", "refId": "A",
"sql": { "sql": {
"columns": [ "columns": [
@ -183,7 +183,7 @@
"urlPath": "" "urlPath": ""
} }
], ],
"title": "Finished buld tasks", "title": "Build tasks",
"transformations": [ "transformations": [
{ {
"id": "convertFieldType", "id": "convertFieldType",
@ -239,6 +239,6 @@
"timezone": "", "timezone": "",
"title": "Build details", "title": "Build details",
"uid": "dmVtrz-4k", "uid": "dmVtrz-4k",
"version": 5, "version": 8,
"weekStart": "" "weekStart": ""
} }

566
grafana-dashbords/Build task details.json
View File

@ -41,12 +41,6 @@
"name": "PostgreSQL", "name": "PostgreSQL",
"version": "1.0.0" "version": "1.0.0"
}, },
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{ {
"type": "panel", "type": "panel",
"id": "table", "id": "table",
@ -229,7 +223,23 @@
}, },
"unit": "s" "unit": "s"
}, },
"overrides": [] "overrides": [
{
"matcher": {
"id": "byName",
"options": "wait time"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "dark-red",
"mode": "fixed"
}
}
]
}
]
}, },
"gridPos": { "gridPos": {
"h": 9, "h": 9,
@ -254,49 +264,6 @@
}, },
"pluginVersion": "9.3.2", "pluginVersion": "9.3.2",
"targets": [ "targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "upload",
"sql": {
"columns": [
{
"parameters": [
{
"name": "start_ts",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "build_node_stats",
"urlPath": ""
},
{ {
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
@ -306,8 +273,8 @@
"format": "table", "format": "table",
"hide": false, "hide": false,
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2", "rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3",
"refId": "build_srpm", "refId": "git_checkout",
"sql": { "sql": {
"columns": [ "columns": [
{ {
@ -335,8 +302,37 @@
"format": "table", "format": "table",
"hide": false, "hide": false,
"rawQuery": true, "rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as git_checkout\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 3", "rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;",
"refId": "git_checkout", "refId": "cas_source_authenticate",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_srpm\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 2",
"refId": "build_srpm",
"sql": { "sql": {
"columns": [ "columns": [
{ {
@ -413,35 +409,6 @@
"limit": 50 "limit": 50
} }
}, },
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as source_authenticate\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 8;",
"refId": "cas_source_authenticate",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{ {
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
@ -499,6 +466,146 @@
], ],
"limit": 50 "limit": 50
} }
},
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as upload\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "upload",
"sql": {
"columns": [
{
"parameters": [
{
"name": "start_ts",
"type": "functionParameter"
}
],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"table": "build_node_stats",
"urlPath": ""
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as build_node_task\nFROM build_node_stats \nWHERE build_task_id = $build_task and stat_name_id = 6",
"refId": "build_node_task",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT end_ts - start_ts as web_node_task\nFROM web_node_stats\nWHERE build_task_id = $build_task and stat_name_id = 0",
"refId": "web_node_task",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT\n finished_at - started_at AS duration \nFROM build_tasks \nWHERE id = $build_task;",
"refId": "task_diration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "__expr__",
"uid": "${DS_EXPRESSION}"
},
"expression": "$task_diration - $build_node_task - $web_node_task",
"hide": false,
"refId": "wait time",
"type": "math"
} }
], ],
"title": "Task duration", "title": "Task duration",
@ -605,7 +712,12 @@
}, },
"frameIndex": 1, "frameIndex": 1,
"showHeader": true, "showHeader": true,
"sortBy": [] "sortBy": [
{
"desc": true,
"displayName": "started"
}
]
}, },
"pluginVersion": "9.3.2", "pluginVersion": "9.3.2",
"targets": [ "targets": [
@ -690,8 +802,7 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green", "color": "green"
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -811,287 +922,6 @@
} }
], ],
"type": "table" "type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"filterable": true,
"inspect": true
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 11,
"w": 24,
"x": 0,
"y": 29
},
"id": 6,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "finished"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT enum.value,\n stats.start_ts * 1000 AS started,\n stats.end_ts * 1000 AS finished,\n stats.end_ts - stats.start_ts AS duration\nFROM build_node_stats AS stats\nINNER JOIN build_node_stats_enum AS enum\n ON stats.stat_name_id = enum.id\nWHERE build_task_id = $build_task",
"refId": "web node stats",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Build node stats",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 17,
"x": 3,
"y": 40
},
"id": 14,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"targets": [
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"rawQuery": true,
"rawSql": "SELECT\n bt.finished_at - bt.started_at AS task_duration \nFROM build_tasks AS bt\nWHERE bt.id = $build_task;",
"refId": "task_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n stats.end_ts - stats.start_ts AS web_node_duration\nFROM web_node_stats AS stats\nWHERE build_task_id = $build_task and stat_name_id=0;",
"refId": "web_node_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n stats.end_ts - stats.start_ts AS build_node_duration\nFROM build_node_stats AS stats\nWHERE build_task_id = $build_task AND stat_name_id=6",
"refId": "build_node_duration",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
},
{
"datasource": {
"type": "__expr__",
"uid": "${DS_EXPRESSION}"
},
"expression": "$task_duration - ($web_node_duration + $build_node_duration)",
"hide": false,
"refId": "Not accounted duration",
"type": "math"
}
],
"title": "Diff",
"type": "stat"
} }
], ],
"schemaVersion": 37, "schemaVersion": 37,
@ -1128,6 +958,6 @@
"timezone": "", "timezone": "",
"title": "Build task details", "title": "Build task details",
"uid": "vtQClqxVk", "uid": "vtQClqxVk",
"version": 27, "version": 34,
"weekStart": "" "weekStart": ""
} }