debugging

This commit is contained in:
Kirill Zhukov 2023-03-16 09:31:09 +01:00
parent 08ec138942
commit 45a6850056
12 changed files with 54 additions and 45 deletions

View File

@ -150,7 +150,7 @@ class APIclient():
response = requests.get( response = requests.get(
url, headers=headers, timeout=self.timeout) url, headers=headers, timeout=self.timeout)
response.raise_for_status() response.raise_for_status()
return self.__parse_test_tasks() return self.__parse_test_tasks(response.json(), build_task_id)
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]], def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
build_task_id: int, build_task_id: int,
@ -161,33 +161,30 @@ class APIclient():
started_raw = task['alts_response']['stats']['started_at'] started_raw = task['alts_response']['stats']['started_at']
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET) started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
stats_raw = task['alts_response']['stats'] stats_raw = task['alts_response']['stats']
results_raw = task['results']['tests'] steps_stats = self.__parse_test_steps_stats(stats_raw)
step_stats = self.__parse_test_steps_stats(
stats_raw, results_raw)
else: else:
started_at = None started_at = None
step_stats = None steps_stats = None
params = { params = {
'id': task['id'], 'id': task['id'],
'build_task_id': build_task_id, 'build_task_id': build_task_id,
'revision': task['revision'], 'revision': task['revision'],
'status': task['status'], 'status': task['status'],
'package_fullname': '_'.join([raw_tasks['package_name'], 'package_fullname': '_'.join([task['package_name'],
raw_tasks['package_version'], task['package_version'],
raw_tasks['package_release']]), task['package_release']]),
'started_at': started_at, 'started_at': started_at,
'step_stats': step_stats 'steps_stats': steps_stats
} }
result.append(TestTask(**params)) result.append(TestTask(**params))
return result return result
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats: def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
teast_steps_params = {} teast_steps_params = {}
for field_name in TestStepsStats.__fields__.keys(): for field_name in TestStepsStats.__fields__.keys():
try: try:
p = stats_raw[field_name] p = stats_raw[field_name]
except KeyError: except KeyError:
continue continue
p['success'] = results_raw[field_name]['success']
teast_steps_params[field_name] = TestStepStat(**p) teast_steps_params[field_name] = TestStepStat(**p)
return TestStepsStats(**teast_steps_params) return TestStepsStats(**teast_steps_params)

View File

@ -54,7 +54,8 @@ class TestStepEnum(IntEnum):
install_package = 0 install_package = 0
stop_enviroment = 1 stop_enviroment = 1
initial_provision = 2 initial_provision = 2
start_enviroment = 3 start_environment = 3
uninstall_package = 4 uninstall_package = 4
initialize_terraform = 5 initialize_terraform = 5
package_integrity_tests = 6 package_integrity_tests = 6
stop_environment = 7

View File

@ -232,24 +232,26 @@ class DB():
return val == 1 return val == 1
def insert_test_task(self, task: TestTaskDB): def insert_test_task(self, task: TestTaskDB):
cur = self.__conn.cursor()
# inserting test task itself # inserting test task itself
sql = ''' sql = '''
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at) INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
VALUES (%s, %s, %s, %s, %s, %s); VALUES
(%s, %s, %s, %s, %s, %s);
''' '''
cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
cur = self.__conn.cursor()
cur.execute(sql, (task.id, task.build_task_id, task.status_id,
task.package_fullname, task.started_at)) task.package_fullname, task.started_at))
# inserting test steps stats
for ss in task.steps_stats: if task.steps_stats:
sql = ''' # inserting test steps stats
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success) for ss in task.steps_stats:
VALUES sql = '''
(%s, %s, %s, %s, %s); INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
''' VALUES
cur.execute(sql, (ss.test_task_id, ss.stat_name_id, (%s, %s, %s, %s);
ss.start_ts, ss.end_ts, ss.success)) '''
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
ss.start_ts, ss.finish_ts))
# commiting changes # commiting changes
self.__conn.commit() self.__conn.commit()

View File

@ -56,6 +56,15 @@ class Extractor:
except Exception as error: # pylint: disable=broad-except except Exception as error: # pylint: disable=broad-except
logging.error('build %s: failed to insert build task %d: %s', logging.error('build %s: failed to insert build task %d: %s',
build.id, build_task.id, error, exc_info=True) build.id, build_task.id, error, exc_info=True)
logging.info(
'getting test tasks for build task %s', build_task.id)
test_tasks = self.api.get_test_tasks(build_task.id)
logging.info('received %d tests tasks', len(test_tasks))
for t in test_tasks:
logging.info(
'build task %s: inserting test task %s', build_task.id, t.id)
self.db.insert_test_task(t.as_db_model())
build_count += 1 build_count += 1
page_num += 1 page_num += 1
return build_count return build_count

View File

@ -1,4 +1,4 @@
from pydantic import BaseModel from pydantic import BaseModel # pylint: disable=no-name-in-module
class SignTaskDB(BaseModel): class SignTaskDB(BaseModel):

View File

@ -6,5 +6,4 @@ from pydantic import BaseModel # pylint: disable=no-name-in-module
class TestStepStat(BaseModel): class TestStepStat(BaseModel):
start_ts: Optional[datetime] = None start_ts: Optional[datetime] = None
end_ts: Optional[datetime] = None finish_ts: Optional[datetime] = None
success: bool

View File

@ -6,5 +6,4 @@ class TestStepStatDB(BaseModel):
test_task_id: int test_task_id: int
stat_name_id: int stat_name_id: int
start_ts: Optional[float] = None start_ts: Optional[float] = None
end_ts: Optional[float] = None finish_ts: Optional[float] = None
success: bool

View File

@ -24,14 +24,13 @@ class TestStepsStats(BaseModel):
continue continue
start_ts = stats.start_ts.timestamp() \ start_ts = stats.start_ts.timestamp() \
if stats.start_ts else None if stats.start_ts else None
end_ts = stats.end_ts.timestamp() \ finish_ts = stats.finish_ts.timestamp() \
if stats.end_ts else None if stats.finish_ts else None
stat_name_id = TestStepEnum[field_name].value stat_name_id = TestStepEnum[field_name].value
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id, test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
stat_name_id=stat_name_id, stat_name_id=stat_name_id,
start_ts=start_ts, start_ts=start_ts,
end_ts=end_ts, finish_ts=finish_ts)
success=stats.success)
result.append(test_step_stat_db) result.append(test_step_stat_db)
return result return result

View File

@ -14,7 +14,7 @@ class TestTask(BaseModel):
status: int status: int
package_fullname: str package_fullname: str
started_at: Optional[datetime] = None started_at: Optional[datetime] = None
steps_stats: TestStepsStats steps_stats: Optional[TestStepsStats] = None
def as_db_model(self) -> TestTaskDB: def as_db_model(self) -> TestTaskDB:
started_at = self.started_at.timestamp() \ started_at = self.started_at.timestamp() \
@ -23,9 +23,9 @@ class TestTask(BaseModel):
'id': self.id, 'id': self.id,
'build_task_id': self.build_task_id, 'build_task_id': self.build_task_id,
'revision': self.revision, 'revision': self.revision,
'status': self.status, 'status_id': self.status,
'package_fullname': self.package_fullname, 'package_fullname': self.package_fullname,
'started_at': started_at, 'started_at': started_at,
'steps_stats': self.step_stats.as_db(self.id) 'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
} }
return TestTaskDB(**params) return TestTaskDB(**params)

View File

@ -13,5 +13,5 @@ class TestTaskDB(BaseModel):
revision: int revision: int
status_id: int status_id: int
package_fullname: str package_fullname: str
started_at: float started_at: Optional[float] = None
steps_stats: List[TestStepStatDB] = None steps_stats: Optional[List[TestStepStatDB]] = None

View File

@ -48,17 +48,17 @@ VALUES
(3, 'start_environment'), (3, 'start_environment'),
(4, 'uninstall_package'), (4, 'uninstall_package'),
(5, 'initialize_terraform'), (5, 'initialize_terraform'),
(6, 'package_integrity_tests'); (6, 'package_integrity_tests'),
(7, 'stop_environment');
-- test_steps -- test_steps
CREATE TABLE test_steps_stats( CREATE TABLE test_steps_stats(
test_task_id INTEGER, test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL, stat_name_id INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
start_ts DOUBLE PRECISION, start_ts DOUBLE PRECISION,
end_ts DOUBLE PRECISION, finish_ts DOUBLE PRECISION
success BOOLEAN
); );
ALTER TABLE test_steps_stats ALTER TABLE test_steps_stats

View File

@ -7,4 +7,7 @@ First version
- Added metrics for build steps - Added metrics for build steps
0.2.1 (2023-03-15) 0.2.1 (2023-03-15)
- Added canceled Build task status - Added canceled Build task status
0.3.0 (IN PROGRESS)
- Added test tasks stats