kzhukov
/
albs_analytics
1
0
Fork 0
Code Issues Pull Requests Releases 6 Activity

debugging

This commit is contained in:
Kirill Zhukov 2023-03-16 09:31:09 +01:00
parent 08ec138942
commit 45a6850056
12 changed files with 54 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
build_analytics/build_analytics/api_client.py
View File

@ -150,7 +150,7 @@ class APIclient():
response = requests.get(
url, headers=headers, timeout=self.timeout)
response.raise_for_status()
return self.__parse_test_tasks()
return self.__parse_test_tasks(response.json(), build_task_id)
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
build_task_id: int,
@ -161,33 +161,30 @@ class APIclient():
started_raw = task['alts_response']['stats']['started_at']
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
stats_raw = task['alts_response']['stats']
results_raw = task['results']['tests']
step_stats = self.__parse_test_steps_stats(
stats_raw, results_raw)
steps_stats = self.__parse_test_steps_stats(stats_raw)
else:
started_at = None
step_stats = None
steps_stats = None
params = {
'id': task['id'],
'build_task_id': build_task_id,
'revision': task['revision'],
'status': task['status'],
'package_fullname': '_'.join([raw_tasks['package_name'],
raw_tasks['package_version'],
raw_tasks['package_release']]),
'package_fullname': '_'.join([task['package_name'],
task['package_version'],
task['package_release']]),
'started_at': started_at,
'step_stats': step_stats
'steps_stats': steps_stats
}
result.append(TestTask(**params))
return result
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats:
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
teast_steps_params = {}
for field_name in TestStepsStats.__fields__.keys():
try:
p = stats_raw[field_name]
except KeyError:
continue
p['success'] = results_raw[field_name]['success']
teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params)

3
build_analytics/build_analytics/const.py
View File

@ -54,7 +54,8 @@ class TestStepEnum(IntEnum):
install_package = 0
stop_enviroment = 1
initial_provision = 2
start_enviroment = 3
start_environment = 3
uninstall_package = 4
initialize_terraform = 5
package_integrity_tests = 6
stop_environment = 7

28
build_analytics/build_analytics/db.py
View File

@ -232,24 +232,26 @@ class DB():
return val == 1
def insert_test_task(self, task: TestTaskDB):
cur = self.__conn.cursor()
# inserting test task itself
sql = '''
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
VALUES (%s, %s, %s, %s, %s, %s);
VALUES
(%s, %s, %s, %s, %s, %s);
'''
cur = self.__conn.cursor()
cur.execute(sql, (task.id, task.build_task_id, task.status_id,
cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
task.package_fullname, task.started_at))
# inserting test steps stats
for ss in task.steps_stats:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
VALUES
(%s, %s, %s, %s, %s);
'''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.end_ts, ss.success))
if task.steps_stats:
# inserting test steps stats
for ss in task.steps_stats:
sql = '''
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
VALUES
(%s, %s, %s, %s);
'''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.finish_ts))
# commiting changes
self.__conn.commit()

9
build_analytics/build_analytics/extractor/extractor.py
View File

@ -56,6 +56,15 @@ class Extractor:
except Exception as error: # pylint: disable=broad-except
logging.error('build %s: failed to insert build task %d: %s',
build.id, build_task.id, error, exc_info=True)
logging.info(
'getting test tasks for build task %s', build_task.id)
test_tasks = self.api.get_test_tasks(build_task.id)
logging.info('received %d tests tasks', len(test_tasks))
for t in test_tasks:
logging.info(
'build task %s: inserting test task %s', build_task.id, t.id)
self.db.insert_test_task(t.as_db_model())
build_count += 1
page_num += 1
return build_count

2
build_analytics/build_analytics/models/sign_task_db.py
View File

@ -1,4 +1,4 @@
from pydantic import BaseModel
from pydantic import BaseModel # pylint: disable=no-name-in-module
class SignTaskDB(BaseModel):

3
build_analytics/build_analytics/models/test_step_stat.py
View File

@ -6,5 +6,4 @@ from pydantic import BaseModel # pylint: disable=no-name-in-module
class TestStepStat(BaseModel):
start_ts: Optional[datetime] = None
end_ts: Optional[datetime] = None
success: bool
finish_ts: Optional[datetime] = None

3
build_analytics/build_analytics/models/test_step_stat_db.py
View File

@ -6,5 +6,4 @@ class TestStepStatDB(BaseModel):
test_task_id: int
stat_name_id: int
start_ts: Optional[float] = None
end_ts: Optional[float] = None
success: bool
finish_ts: Optional[float] = None

7
build_analytics/build_analytics/models/test_steps_stats.py
View File

@ -24,14 +24,13 @@ class TestStepsStats(BaseModel):
continue
start_ts = stats.start_ts.timestamp() \
if stats.start_ts else None
end_ts = stats.end_ts.timestamp() \
if stats.end_ts else None
finish_ts = stats.finish_ts.timestamp() \
if stats.finish_ts else None
stat_name_id = TestStepEnum[field_name].value
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
stat_name_id=stat_name_id,
start_ts=start_ts,
end_ts=end_ts,
success=stats.success)
finish_ts=finish_ts)
result.append(test_step_stat_db)
return result

6
build_analytics/build_analytics/models/test_task.py
View File

@ -14,7 +14,7 @@ class TestTask(BaseModel):
status: int
package_fullname: str
started_at: Optional[datetime] = None
steps_stats: TestStepsStats
steps_stats: Optional[TestStepsStats] = None
def as_db_model(self) -> TestTaskDB:
started_at = self.started_at.timestamp() \
@ -23,9 +23,9 @@ class TestTask(BaseModel):
'id': self.id,
'build_task_id': self.build_task_id,
'revision': self.revision,
'status': self.status,
'status_id': self.status,
'package_fullname': self.package_fullname,
'started_at': started_at,
'steps_stats': self.step_stats.as_db(self.id)
'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
}
return TestTaskDB(**params)

4
build_analytics/build_analytics/models/test_task_db.py
View File

@ -13,5 +13,5 @@ class TestTaskDB(BaseModel):
revision: int
status_id: int
package_fullname: str
started_at: float
steps_stats: List[TestStepStatDB] = None
started_at: Optional[float] = None
steps_stats: Optional[List[TestStepStatDB]] = None

10
build_analytics/migrations/3.sql
View File

@ -48,17 +48,17 @@ VALUES
(3, 'start_environment'),
(4, 'uninstall_package'),
(5, 'initialize_terraform'),
(6, 'package_integrity_tests');
(6, 'package_integrity_tests'),
(7, 'stop_environment');
-- test_steps
CREATE TABLE test_steps_stats(
test_task_id INTEGER,
stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL,
test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
stat_name_id INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
start_ts DOUBLE PRECISION,
end_ts DOUBLE PRECISION,
success BOOLEAN
finish_ts DOUBLE PRECISION
);
ALTER TABLE test_steps_stats

5
build_analytics/releases.txt
View File

@ -7,4 +7,7 @@ First version
- Added metrics for build steps
0.2.1 (2023-03-15)
- Added canceled Build task status
- Added canceled Build task status
0.3.0 (IN PROGRESS)
- Added test tasks stats