kzhukov
/
albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

- Added new try/catch segments 

- Added tz info to test step stats timestamps
- Increased oldest_to_update_days parameter
This commit is contained in:
Kirill Zhukov 2023-03-20 19:29:45 +01:00
parent 313d4a4d2a
commit 4d5ffcc74f
6 changed files with 918 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
build_analytics/build_analytics/api_client.py
View File

@ -158,9 +158,17 @@ class APIclient():
result: List[TestTask] = []
for task in raw_tasks:
if task['alts_response']:
try:
started_raw = task['alts_response']['stats']['started_at']
except KeyError:
started_at = None
else:
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
try:
stats_raw = task['alts_response']['stats']
except KeyError:
steps_stats = None
else:
steps_stats = self.__parse_test_steps_stats(stats_raw)
else:
started_at = None
@ -176,6 +184,7 @@ class APIclient():
'started_at': started_at,
'steps_stats': steps_stats
}
result.append(TestTask(**params))
return result
@ -186,5 +195,9 @@ class APIclient():
p = stats_raw[field_name]
except KeyError:
continue
# there are must be a better way
for k in ['start_ts', 'finish_ts']:
if k in p:
p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params)

2
build_analytics/build_analytics/db.py
View File

@ -272,7 +272,7 @@ class DB():
WHERE tt.status_id < 3 AND bt.started_at > %s;
'''
cur.execute(sql, (not_before.timestamp(),))
logging.info('raw SQL query: %s', cur.query)
logging.debug('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()]
return result

2
build_analytics/build_analytics/models/extractor_config.py
View File

@ -11,7 +11,7 @@ LOG_FILE_DEFAULT = '/tmp/extractor.log'
API_DEFAULT = 30
SCRAPE_INTERVAL_DEFAULT = 3600
START_FROM_DEFAULT = 5808
OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=3)
OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=7)
class ExtractorConfig(BaseModel):

4
build_analytics/config_default.yml
View File

@ -66,6 +66,6 @@ start_from:
# oldest_to_update
# oldest (in days) unfinished object (build/task/step...) that we will try to update
# required: false
# default: 3
oldest_to_update_days: 3
# default: 7
oldest_to_update_days: 7

92
build_analytics/grafana-dashbords/Build analytics.json
View File

@ -3304,7 +3304,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3459,7 +3460,7 @@
"type": "row"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@ -3467,7 +3468,10 @@
"y": 15
},
"id": 39,
"panels": [
"panels": [],
"title": "Build tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
@ -3480,17 +3484,18 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3503,7 +3508,7 @@
{
"matcher": {
"id": "byName",
"options": "duration"
"options": "task_duration"
},
"properties": [
{
@ -3512,7 +3517,7 @@
},
{
"id": "custom.width",
"value": 103
"value": 124
}
]
},
@ -3530,6 +3535,10 @@
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n"
}
]
},
{
"id": "custom.width",
"value": 81
}
]
},
@ -3553,7 +3562,7 @@
"properties": [
{
"id": "custom.width",
"value": 108
"value": 274
}
]
},
@ -3589,6 +3598,54 @@
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "arch"
},
"properties": [
{
"id": "custom.width",
"value": 104
}
]
},
{
"matcher": {
"id": "byName",
"options": "finished"
},
"properties": [
{
"id": "custom.width",
"value": 220
}
]
},
{
"matcher": {
"id": "byName",
"options": "started"
},
"properties": [
{
"id": "custom.width",
"value": 197
}
]
},
{
"matcher": {
"id": "byName",
"options": "status"
},
"properties": [
{
"id": "custom.width",
"value": 152
}
]
}
]
},
@ -3724,17 +3781,18 @@
"mode": "thresholds"
},
"custom": {
"align": "auto",
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": true
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -3806,7 +3864,7 @@
},
{
"id": "custom.width",
"value": 102
"value": 80
}
]
},
@ -3818,7 +3876,7 @@
"properties": [
{
"id": "custom.width",
"value": 75
"value": 294
}
]
},
@ -3921,10 +3979,6 @@
"type": "table"
}
],
"title": "Build tasks",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 37,
"style": "dark",
@ -3940,6 +3994,6 @@
"timezone": "",
"title": "Build analytics",
"uid": "02mg4oxVk",
"version": 63,
"version": 64,
"weekStart": ""
}

135
build_analytics/grafana-dashbords/Build details.json
View File

@ -1,8 +1,8 @@
{
"__inputs": [
{
"name": "DS_POSTGRESQL",
"label": "PostgreSQL",
"name": "DS_ALBS_ANALYTICS",
"label": "albs_analytics",
"description": "",
"type": "datasource",
"pluginId": "postgres",
@ -21,7 +21,7 @@
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "9.3.2"
"version": "9.3.6"
},
{
"type": "panel",
@ -93,7 +93,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -201,13 +201,13 @@
}
]
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -266,7 +266,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -396,7 +396,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -425,7 +425,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -454,7 +454,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -483,7 +483,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -512,7 +512,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -545,7 +545,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -735,7 +735,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -764,7 +764,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -793,7 +793,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -822,7 +822,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -851,7 +851,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -884,7 +884,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "Averages for build tasks with \"completed\"\nstatus",
"fieldConfig": {
@ -928,13 +928,13 @@
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -971,7 +971,7 @@
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -1009,7 +1009,7 @@
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -1046,7 +1046,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1079,7 +1079,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "How much time were spent in queue. \n\nStats for \"completed\" build tasks only",
"fieldConfig": {
@ -1159,13 +1159,13 @@
},
"textMode": "auto"
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -1277,7 +1277,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -1454,7 +1454,7 @@
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -1497,7 +1497,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1526,7 +1526,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1555,7 +1555,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1584,7 +1584,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1613,7 +1613,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1642,7 +1642,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1671,13 +1671,13 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ",
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession",
"sql": {
"columns": [
@ -1704,7 +1704,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -1752,13 +1752,13 @@
},
"showUnfilled": false
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -1801,7 +1801,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1830,7 +1830,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1859,7 +1859,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1888,7 +1888,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1917,7 +1917,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1946,7 +1946,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
@ -1975,14 +1975,14 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"format": "table",
"hide": false,
"rawQuery": true,
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 1 AND build_tasks.build_id = $build_id ",
"refId": "packages_procession",
"rawSql": "SELECT \n SUM(end_ts - start_ts) AS packages_processing\nFROM web_node_stats AS bs \nJOIN build_tasks \n ON bs.build_task_id = build_tasks.id\nWHERE stat_name_id = 2 AND build_tasks.build_id = $build_id ",
"refId": "packages_processing",
"sql": {
"columns": [
{
@ -2006,7 +2006,7 @@
"type": "bargauge"
},
{
"collapsed": true,
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@ -2014,11 +2014,14 @@
"y": 21
},
"id": 18,
"panels": [
"panels": [],
"title": "Build tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"description": "",
"fieldConfig": {
@ -2037,7 +2040,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
}
]
}
@ -2079,7 +2083,7 @@
"h": 9,
"w": 24,
"x": 0,
"y": 3
"y": 22
},
"id": 2,
"options": {
@ -2098,13 +2102,13 @@
}
]
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -2162,7 +2166,7 @@
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"fieldConfig": {
"defaults": {
@ -2180,7 +2184,8 @@
"mode": "absolute",
"steps": [
{
"color": "green"
"color": "green",
"value": null
},
{
"color": "red",
@ -2286,7 +2291,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 12
"y": 31
},
"id": 25,
"options": {
@ -2305,13 +2310,13 @@
}
]
},
"pluginVersion": "9.3.2",
"pluginVersion": "9.3.6",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"editorMode": "code",
"fields": [
@ -2367,10 +2372,6 @@
"type": "table"
}
],
"title": "Build tasks",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 37,
"style": "dark",
@ -2381,7 +2382,7 @@
"current": {},
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
"uid": "${DS_ALBS_ANALYTICS}"
},
"definition": "SELECT id\nFROM builds\nORDER BY id DESC",
"hide": 0,
@ -2406,6 +2407,6 @@
"timezone": "",
"title": "Build details",
"uid": "dmVtrz-4k",
"version": 21,
"version": 5,
"weekStart": ""
}