From 45a68500564761c439125d808d442894528ac3fa Mon Sep 17 00:00:00 2001 From: Kirill Zhukov Date: Thu, 16 Mar 2023 09:31:09 +0100 Subject: [PATCH] debugging --- build_analytics/build_analytics/api_client.py | 19 ++++++------- build_analytics/build_analytics/const.py | 3 +- build_analytics/build_analytics/db.py | 28 ++++++++++--------- .../build_analytics/extractor/extractor.py | 9 ++++++ .../build_analytics/models/sign_task_db.py | 2 +- .../build_analytics/models/test_step_stat.py | 3 +- .../models/test_step_stat_db.py | 3 +- .../models/test_steps_stats.py | 7 ++--- .../build_analytics/models/test_task.py | 6 ++-- .../build_analytics/models/test_task_db.py | 4 +-- build_analytics/migrations/3.sql | 10 +++---- build_analytics/releases.txt | 5 +++- 12 files changed, 54 insertions(+), 45 deletions(-) diff --git a/build_analytics/build_analytics/api_client.py b/build_analytics/build_analytics/api_client.py index 2b112e0..e16c7d5 100644 --- a/build_analytics/build_analytics/api_client.py +++ b/build_analytics/build_analytics/api_client.py @@ -150,7 +150,7 @@ class APIclient(): response = requests.get( url, headers=headers, timeout=self.timeout) response.raise_for_status() - return self.__parse_test_tasks() + return self.__parse_test_tasks(response.json(), build_task_id) def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]], build_task_id: int, @@ -161,33 +161,30 @@ class APIclient(): started_raw = task['alts_response']['stats']['started_at'] started_at = datetime.fromisoformat(started_raw+TZ_OFFSET) stats_raw = task['alts_response']['stats'] - results_raw = task['results']['tests'] - step_stats = self.__parse_test_steps_stats( - stats_raw, results_raw) + steps_stats = self.__parse_test_steps_stats(stats_raw) else: started_at = None - step_stats = None + steps_stats = None params = { 'id': task['id'], 'build_task_id': build_task_id, 'revision': task['revision'], 'status': task['status'], - 'package_fullname': '_'.join([raw_tasks['package_name'], - raw_tasks['package_version'], - raw_tasks['package_release']]), + 'package_fullname': '_'.join([task['package_name'], + task['package_version'], + task['package_release']]), 'started_at': started_at, - 'step_stats': step_stats + 'steps_stats': steps_stats } result.append(TestTask(**params)) return result - def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats: + def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats: teast_steps_params = {} for field_name in TestStepsStats.__fields__.keys(): try: p = stats_raw[field_name] except KeyError: continue - p['success'] = results_raw[field_name]['success'] teast_steps_params[field_name] = TestStepStat(**p) return TestStepsStats(**teast_steps_params) diff --git a/build_analytics/build_analytics/const.py b/build_analytics/build_analytics/const.py index d77ff24..37367ab 100644 --- a/build_analytics/build_analytics/const.py +++ b/build_analytics/build_analytics/const.py @@ -54,7 +54,8 @@ class TestStepEnum(IntEnum): install_package = 0 stop_enviroment = 1 initial_provision = 2 - start_enviroment = 3 + start_environment = 3 uninstall_package = 4 initialize_terraform = 5 package_integrity_tests = 6 + stop_environment = 7 diff --git a/build_analytics/build_analytics/db.py b/build_analytics/build_analytics/db.py index 0b60d0d..1a4ab0f 100644 --- a/build_analytics/build_analytics/db.py +++ b/build_analytics/build_analytics/db.py @@ -232,24 +232,26 @@ class DB(): return val == 1 def insert_test_task(self, task: TestTaskDB): + cur = self.__conn.cursor() # inserting test task itself sql = ''' INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at) - VALUES (%s, %s, %s, %s, %s, %s); + VALUES + (%s, %s, %s, %s, %s, %s); ''' - - cur = self.__conn.cursor() - cur.execute(sql, (task.id, task.build_task_id, task.status_id, + cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id, task.package_fullname, task.started_at)) - # inserting test steps stats - for ss in task.steps_stats: - sql = ''' - INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success) - VALUES - (%s, %s, %s, %s, %s); - ''' - cur.execute(sql, (ss.test_task_id, ss.stat_name_id, - ss.start_ts, ss.end_ts, ss.success)) + + if task.steps_stats: + # inserting test steps stats + for ss in task.steps_stats: + sql = ''' + INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts) + VALUES + (%s, %s, %s, %s); + ''' + cur.execute(sql, (ss.test_task_id, ss.stat_name_id, + ss.start_ts, ss.finish_ts)) # commiting changes self.__conn.commit() diff --git a/build_analytics/build_analytics/extractor/extractor.py b/build_analytics/build_analytics/extractor/extractor.py index 4702810..7603a44 100644 --- a/build_analytics/build_analytics/extractor/extractor.py +++ b/build_analytics/build_analytics/extractor/extractor.py @@ -56,6 +56,15 @@ class Extractor: except Exception as error: # pylint: disable=broad-except logging.error('build %s: failed to insert build task %d: %s', build.id, build_task.id, error, exc_info=True) + + logging.info( + 'getting test tasks for build task %s', build_task.id) + test_tasks = self.api.get_test_tasks(build_task.id) + logging.info('received %d tests tasks', len(test_tasks)) + for t in test_tasks: + logging.info( + 'build task %s: inserting test task %s', build_task.id, t.id) + self.db.insert_test_task(t.as_db_model()) build_count += 1 page_num += 1 return build_count diff --git a/build_analytics/build_analytics/models/sign_task_db.py b/build_analytics/build_analytics/models/sign_task_db.py index 60281f2..e6d47af 100644 --- a/build_analytics/build_analytics/models/sign_task_db.py +++ b/build_analytics/build_analytics/models/sign_task_db.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel # pylint: disable=no-name-in-module class SignTaskDB(BaseModel): diff --git a/build_analytics/build_analytics/models/test_step_stat.py b/build_analytics/build_analytics/models/test_step_stat.py index 2632646..9dcc598 100644 --- a/build_analytics/build_analytics/models/test_step_stat.py +++ b/build_analytics/build_analytics/models/test_step_stat.py @@ -6,5 +6,4 @@ from pydantic import BaseModel # pylint: disable=no-name-in-module class TestStepStat(BaseModel): start_ts: Optional[datetime] = None - end_ts: Optional[datetime] = None - success: bool + finish_ts: Optional[datetime] = None diff --git a/build_analytics/build_analytics/models/test_step_stat_db.py b/build_analytics/build_analytics/models/test_step_stat_db.py index ddae876..52cc090 100644 --- a/build_analytics/build_analytics/models/test_step_stat_db.py +++ b/build_analytics/build_analytics/models/test_step_stat_db.py @@ -6,5 +6,4 @@ class TestStepStatDB(BaseModel): test_task_id: int stat_name_id: int start_ts: Optional[float] = None - end_ts: Optional[float] = None - success: bool + finish_ts: Optional[float] = None diff --git a/build_analytics/build_analytics/models/test_steps_stats.py b/build_analytics/build_analytics/models/test_steps_stats.py index 5518982..a783b91 100644 --- a/build_analytics/build_analytics/models/test_steps_stats.py +++ b/build_analytics/build_analytics/models/test_steps_stats.py @@ -24,14 +24,13 @@ class TestStepsStats(BaseModel): continue start_ts = stats.start_ts.timestamp() \ if stats.start_ts else None - end_ts = stats.end_ts.timestamp() \ - if stats.end_ts else None + finish_ts = stats.finish_ts.timestamp() \ + if stats.finish_ts else None stat_name_id = TestStepEnum[field_name].value test_step_stat_db = TestStepStatDB(test_task_id=test_task_id, stat_name_id=stat_name_id, start_ts=start_ts, - end_ts=end_ts, - success=stats.success) + finish_ts=finish_ts) result.append(test_step_stat_db) return result diff --git a/build_analytics/build_analytics/models/test_task.py b/build_analytics/build_analytics/models/test_task.py index 1b87fea..e7abe3b 100644 --- a/build_analytics/build_analytics/models/test_task.py +++ b/build_analytics/build_analytics/models/test_task.py @@ -14,7 +14,7 @@ class TestTask(BaseModel): status: int package_fullname: str started_at: Optional[datetime] = None - steps_stats: TestStepsStats + steps_stats: Optional[TestStepsStats] = None def as_db_model(self) -> TestTaskDB: started_at = self.started_at.timestamp() \ @@ -23,9 +23,9 @@ class TestTask(BaseModel): 'id': self.id, 'build_task_id': self.build_task_id, 'revision': self.revision, - 'status': self.status, + 'status_id': self.status, 'package_fullname': self.package_fullname, 'started_at': started_at, - 'steps_stats': self.step_stats.as_db(self.id) + 'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None } return TestTaskDB(**params) diff --git a/build_analytics/build_analytics/models/test_task_db.py b/build_analytics/build_analytics/models/test_task_db.py index 736d054..c1e281a 100644 --- a/build_analytics/build_analytics/models/test_task_db.py +++ b/build_analytics/build_analytics/models/test_task_db.py @@ -13,5 +13,5 @@ class TestTaskDB(BaseModel): revision: int status_id: int package_fullname: str - started_at: float - steps_stats: List[TestStepStatDB] = None + started_at: Optional[float] = None + steps_stats: Optional[List[TestStepStatDB]] = None diff --git a/build_analytics/migrations/3.sql b/build_analytics/migrations/3.sql index cc2b688..8ebb3a5 100644 --- a/build_analytics/migrations/3.sql +++ b/build_analytics/migrations/3.sql @@ -48,17 +48,17 @@ VALUES (3, 'start_environment'), (4, 'uninstall_package'), (5, 'initialize_terraform'), - (6, 'package_integrity_tests'); + (6, 'package_integrity_tests'), + (7, 'stop_environment'); -- test_steps CREATE TABLE test_steps_stats( - test_task_id INTEGER, - stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL, + test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE, + stat_name_id INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL, start_ts DOUBLE PRECISION, - end_ts DOUBLE PRECISION, - success BOOLEAN + finish_ts DOUBLE PRECISION ); ALTER TABLE test_steps_stats diff --git a/build_analytics/releases.txt b/build_analytics/releases.txt index 236fc65..0d8799a 100644 --- a/build_analytics/releases.txt +++ b/build_analytics/releases.txt @@ -7,4 +7,7 @@ First version - Added metrics for build steps 0.2.1 (2023-03-15) - - Added canceled Build task status \ No newline at end of file + - Added canceled Build task status + +0.3.0 (IN PROGRESS) + - Added test tasks stats \ No newline at end of file