kzhukov/albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

- Added new try/catch segments

Browse Source 
- Added tz info to test step stats timestamps
- Increased oldest_to_update_days parameter
This commit is contained in:
Kirill Zhukov 2023-03-20 19:29:45 +01:00
parent 313d4a4d2a
commit 4d5ffcc74f
6 changed files with 918 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
build_analytics/build_analytics/api_client.py
View File

@ -158,10 +158,18 @@ class APIclient():
result: List[TestTask] = [] result: List[TestTask] = []
for task in raw_tasks: for task in raw_tasks:
if task['alts_response']: if task['alts_response']:
started_raw = task['alts_response']['stats']['started_at'] try:
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET) started_raw = task['alts_response']['stats']['started_at']
stats_raw = task['alts_response']['stats'] except KeyError:
steps_stats = self.__parse_test_steps_stats(stats_raw) started_at = None
else:
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
try:
stats_raw = task['alts_response']['stats']
except KeyError:
steps_stats = None
else:
steps_stats = self.__parse_test_steps_stats(stats_raw)
else: else:
started_at = None started_at = None
steps_stats = None steps_stats = None
@ -176,6 +184,7 @@ class APIclient():
'started_at': started_at, 'started_at': started_at,
'steps_stats': steps_stats 'steps_stats': steps_stats
} }
result.append(TestTask(**params)) result.append(TestTask(**params))
return result return result
@ -186,5 +195,9 @@ class APIclient():
p = stats_raw[field_name] p = stats_raw[field_name]
except KeyError: except KeyError:
continue continue
# there are must be a better way
for k in ['start_ts', 'finish_ts']:
if k in p:
p[k] = datetime.fromisoformat(p[k]+TZ_OFFSET)
teast_steps_params[field_name] = TestStepStat(**p) teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params) return TestStepsStats(**teast_steps_params)

2
build_analytics/build_analytics/db.py
View File

@ -272,7 +272,7 @@ class DB():
WHERE tt.status_id < 3 AND bt.started_at > %s; WHERE tt.status_id < 3 AND bt.started_at > %s;
''' '''
cur.execute(sql, (not_before.timestamp(),)) cur.execute(sql, (not_before.timestamp(),))
logging.info('raw SQL query: %s', cur.query) logging.debug('raw SQL query: %s', cur.query)
result = [int(row[0]) for row in cur.fetchall()] result = [int(row[0]) for row in cur.fetchall()]
return result return result

2
build_analytics/build_analytics/models/extractor_config.py
View File

@ -11,7 +11,7 @@ LOG_FILE_DEFAULT = '/tmp/extractor.log'
API_DEFAULT = 30 API_DEFAULT = 30
SCRAPE_INTERVAL_DEFAULT = 3600 SCRAPE_INTERVAL_DEFAULT = 3600
START_FROM_DEFAULT = 5808 START_FROM_DEFAULT = 5808
OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=3) OLDEST_TO_UPDATE_DEFAULT = datetime.now().astimezone() - timedelta(days=7)
class ExtractorConfig(BaseModel): class ExtractorConfig(BaseModel):

4
build_analytics/config_default.yml
View File

@ -66,6 +66,6 @@ start_from:
# oldest_to_update # oldest_to_update
# oldest (in days) unfinished object (build/task/step...) that we will try to update # oldest (in days) unfinished object (build/task/step...) that we will try to update
# required: false # required: false
# default: 3 # default: 7
oldest_to_update_days: 3 oldest_to_update_days: 7

928
build_analytics/grafana-dashbords/Build analytics.json
View File

@ -3304,7 +3304,8 @@
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{ {
"color": "green" "color": "green",
"value": null
}, },
{ {
"color": "red", "color": "red",
@ -3459,7 +3460,7 @@
"type": "row" "type": "row"
}, },
{ {
"collapsed": true, "collapsed": false,
"gridPos": { "gridPos": {
"h": 1, "h": 1,
"w": 24, "w": 24,
@ -3467,462 +3468,515 @@
"y": 15 "y": 15
}, },
"id": 39, "id": 39,
"panels": [ "panels": [],
"title": "Build tasks",
"type": "row"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "task_duration"
},
"properties": [
{
"id": "unit",
"value": "s"
},
{
"id": "custom.width",
"value": 124
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n"
}
]
},
{
"id": "custom.width",
"value": 81
}
]
},
{
"matcher": {
"id": "byName",
"options": "build_id"
},
"properties": [
{
"id": "custom.width",
"value": 88
}
]
},
{
"matcher": {
"id": "byName",
"options": "name"
},
"properties": [
{
"id": "custom.width",
"value": 274
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "build id"
},
"properties": [
{
"id": "custom.width",
"value": 89
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "arch"
},
"properties": [
{
"id": "custom.width",
"value": 104
}
]
},
{
"matcher": {
"id": "byName",
"options": "finished"
},
"properties": [
{
"id": "custom.width",
"value": 220
}
]
},
{
"matcher": {
"id": "byName",
"options": "started"
},
"properties": [
{
"id": "custom.width",
"value": 197
}
]
},
{
"matcher": {
"id": "byName",
"options": "status"
},
"properties": [
{
"id": "custom.width",
"value": 152
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 16
},
"id": 37,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "task_duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{ {
"cacheDurationSeconds": 300,
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
"uid": "${DS_POSTGRESQL}" "uid": "${DS_POSTGRESQL}"
}, },
"description": "", "editorMode": "code",
"fieldConfig": { "fields": [
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"filterable": true,
"inspect": true
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "duration"
},
"properties": [
{
"id": "unit",
"value": "s"
},
{
"id": "custom.width",
"value": 103
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "build_id"
},
"properties": [
{
"id": "custom.width",
"value": 88
}
]
},
{
"matcher": {
"id": "byName",
"options": "name"
},
"properties": [
{
"id": "custom.width",
"value": 108
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": ".*duration"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "build id"
},
"properties": [
{
"id": "custom.width",
"value": 89
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n"
}
]
}
]
}
]
},
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 16
},
"id": 37,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "task_duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{ {
"cacheDurationSeconds": 300, "jsonPath": ""
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"hide": false,
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": " SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS task_duration\n FROM build_tasks AS bt\n JOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\n JOIN arch_enum on bt.arch_id = arch_enum.id \n WHERE $__unixEpochFilter(started_at)",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"format": "table",
"hide": true,
"rawQuery": true,
"rawSql": "SELECT\n bt.id, \n bt.build_id,\n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) \nGROUP BY bt.id;",
"refId": "B",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
}
} }
], ],
"title": "All build tasks", "format": "table",
"transformations": [ "hide": false,
{ "method": "GET",
"id": "convertFieldType", "queryParams": "",
"options": { "rawQuery": true,
"conversions": [ "rawSql": " SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n status_enum.value AS status,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n bt.finished_at - bt.started_at AS task_duration\n FROM build_tasks AS bt\n JOIN build_task_status_enum AS status_enum ON bt.status_id = status_enum.id\n JOIN arch_enum on bt.arch_id = arch_enum.id \n WHERE $__unixEpochFilter(started_at)",
{ "refId": "A",
"destinationType": "time", "sql": {
"targetField": "started" "columns": [
}, {
{ "parameters": [],
"destinationType": "time", "type": "function"
"targetField": "finished"
}
],
"fields": {}
} }
}, ],
{ "groupBy": [
"id": "joinByField", {
"options": { "property": {
"byField": "id", "type": "string"
"mode": "outer" },
"type": "groupBy"
} }
} ],
], "limit": 50
"type": "table" },
"urlPath": ""
}, },
{ {
"datasource": { "datasource": {
"type": "postgres", "type": "postgres",
"uid": "${DS_POSTGRESQL}" "uid": "${DS_POSTGRESQL}"
}, },
"fieldConfig": { "editorMode": "code",
"defaults": { "format": "table",
"color": { "hide": true,
"mode": "thresholds" "rawQuery": true,
}, "rawSql": "SELECT\n bt.id, \n bt.build_id,\n SUM(bt.finished_at - bt.started_at) AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\nFROM build_tasks AS bt\nWHERE $__unixEpochFilter(started_at) \nGROUP BY bt.id;",
"custom": { "refId": "B",
"align": "auto", "sql": {
"displayMode": "auto", "columns": [
"filterable": true, {
"inspect": true "parameters": [],
}, "type": "function"
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
} }
}, ],
"overrides": [ "groupBy": [
{ {
"matcher": { "property": {
"id": "byRegexp", "type": "string"
"options": "(.*duration)|(wait time)"
}, },
"properties": [ "type": "groupBy"
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 70
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "arch"
},
"properties": [
{
"id": "custom.width",
"value": 111
}
]
},
{
"matcher": {
"id": "byName",
"options": "build id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n"
}
]
},
{
"id": "custom.width",
"value": 102
}
]
},
{
"matcher": {
"id": "byName",
"options": "name"
},
"properties": [
{
"id": "custom.width",
"value": 75
}
]
},
{
"matcher": {
"id": "byName",
"options": "task duration"
},
"properties": [
{
"id": "custom.width",
"value": 100
}
]
} }
] ],
}, "limit": 50
"gridPos": { }
"h": 10,
"w": 24,
"x": 0,
"y": 25
},
"id": 43,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "task duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE $__unixEpochFilter(started_at) AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE $__unixEpochFilter(started_at) AND status_id = 2",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Completed build tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
} }
], ],
"title": "Build tasks", "title": "All build tasks",
"type": "row" "transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
},
{
"id": "joinByField",
"options": {
"byField": "id",
"mode": "outer"
}
}
],
"type": "table"
},
{
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"displayMode": "auto",
"filterable": true,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "(.*duration)|(wait time)"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.width",
"value": 70
},
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/vtQClqxVk/build-task-details?orgId=1&var-build_task=${__value.raw}\n\n"
}
]
}
]
},
{
"matcher": {
"id": "byName",
"options": "arch"
},
"properties": [
{
"id": "custom.width",
"value": 111
}
]
},
{
"matcher": {
"id": "byName",
"options": "build id"
},
"properties": [
{
"id": "links",
"value": [
{
"title": "",
"url": "/d/dmVtrz-4k/build-details?orgId=1&var-build_id=${__value.raw}\n\n\n"
}
]
},
{
"id": "custom.width",
"value": 80
}
]
},
{
"matcher": {
"id": "byName",
"options": "name"
},
"properties": [
{
"id": "custom.width",
"value": 294
}
]
},
{
"matcher": {
"id": "byName",
"options": "task duration"
},
"properties": [
{
"id": "custom.width",
"value": 100
}
]
}
]
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 25
},
"id": 43,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "task duration"
}
]
},
"pluginVersion": "9.3.2",
"targets": [
{
"cacheDurationSeconds": 300,
"datasource": {
"type": "postgres",
"uid": "${DS_POSTGRESQL}"
},
"editorMode": "code",
"fields": [
{
"jsonPath": ""
}
],
"format": "table",
"method": "GET",
"queryParams": "",
"rawQuery": true,
"rawSql": "SELECT\n bt.id,\n bt.build_id AS \"build id\",\n bt.name,\n arch_enum.value AS arch,\n bt.started_at * 1000 AS started,\n bt.finished_at * 1000 AS finished,\n task_duration AS \"task duration\",\n web_node_duration AS \"web node duration\",\n build_node_duration AS \"build node duration\",\n task_duration - web_node_duration - build_node_duration as \"wait time\"\n FROM build_tasks AS bt\n JOIN arch_enum on bt.arch_id = arch_enum.id \n LEFT JOIN \n (\n SELECT\n bt.id,\n bt.finished_at - bt.started_at AS task_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM web_node_stats AS ws \n WHERE ws.build_task_id = bt.id AND stat_name_id=0) AS web_node_duration,\n (SELECT SUM(end_ts - start_ts) \n FROM build_node_stats AS bs \n WHERE bs.build_task_id = bt.id AND stat_name_id=6) AS build_node_duration\n FROM build_tasks AS bt\n WHERE $__unixEpochFilter(started_at) AND status_id = 2\n GROUP BY bt.id\n ) as d\n ON bt.id = d.id\n WHERE $__unixEpochFilter(started_at) AND status_id = 2",
"refId": "A",
"sql": {
"columns": [
{
"parameters": [],
"type": "function"
}
],
"groupBy": [
{
"property": {
"type": "string"
},
"type": "groupBy"
}
],
"limit": 50
},
"urlPath": ""
}
],
"title": "Completed build tasks",
"transformations": [
{
"id": "convertFieldType",
"options": {
"conversions": [
{
"destinationType": "time",
"targetField": "started"
},
{
"destinationType": "time",
"targetField": "finished"
}
],
"fields": {}
}
}
],
"type": "table"
} }
], ],
"refresh": false, "refresh": false,
@ -3940,6 +3994,6 @@
"timezone": "", "timezone": "",
"title": "Build analytics", "title": "Build analytics",
"uid": "02mg4oxVk", "uid": "02mg4oxVk",
"version": 63, "version": 64,
"weekStart": "" "weekStart": ""
} }

811
build_analytics/grafana-dashbords/Build details.json
View File

File diff suppressed because it is too large Load Diff