added api and db functions
This commit is contained in:
parent
a93165420b
commit
08ec138942
@ -1,16 +1,19 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
import logging
|
import logging
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
from typing import Dict, List
|
from typing import Dict, List, Any
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
|
||||||
|
|
||||||
from .models.build import Build
|
from .models.build import Build
|
||||||
from .models.build_task import BuildTask
|
from .models.build_task import BuildTask
|
||||||
from .models.build_node_stats import BuildNodeStats
|
from .models.build_node_stats import BuildNodeStats
|
||||||
from .models.build_stat import BuildStat
|
from .models.build_stat import BuildStat
|
||||||
from .models.web_node_stats import WebNodeStats
|
from .models.web_node_stats import WebNodeStats
|
||||||
|
from .models.test_task import TestTask
|
||||||
|
from .models.test_steps_stats import TestStepsStats
|
||||||
|
from .models.test_step_stat import TestStepStat
|
||||||
|
|
||||||
TZ_OFFSET = '+00:00'
|
TZ_OFFSET = '+00:00'
|
||||||
|
|
||||||
@ -138,3 +141,53 @@ class APIclient():
|
|||||||
'build_tasks': build_tasks}
|
'build_tasks': build_tasks}
|
||||||
|
|
||||||
return Build(**params)
|
return Build(**params)
|
||||||
|
|
||||||
|
def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
|
||||||
|
ep = f'/api/v1/tests/{build_task_id}/latest'
|
||||||
|
url = urljoin(self.api_root, ep)
|
||||||
|
headers = {'accept': 'application/json'}
|
||||||
|
|
||||||
|
response = requests.get(
|
||||||
|
url, headers=headers, timeout=self.timeout)
|
||||||
|
response.raise_for_status()
|
||||||
|
return self.__parse_test_tasks()
|
||||||
|
|
||||||
|
def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
|
||||||
|
build_task_id: int,
|
||||||
|
started_at: str = None) -> List[TestTask]:
|
||||||
|
result: List[TestTask] = []
|
||||||
|
for task in raw_tasks:
|
||||||
|
if task['alts_response']:
|
||||||
|
started_raw = task['alts_response']['stats']['started_at']
|
||||||
|
started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
|
||||||
|
stats_raw = task['alts_response']['stats']
|
||||||
|
results_raw = task['results']['tests']
|
||||||
|
step_stats = self.__parse_test_steps_stats(
|
||||||
|
stats_raw, results_raw)
|
||||||
|
else:
|
||||||
|
started_at = None
|
||||||
|
step_stats = None
|
||||||
|
params = {
|
||||||
|
'id': task['id'],
|
||||||
|
'build_task_id': build_task_id,
|
||||||
|
'revision': task['revision'],
|
||||||
|
'status': task['status'],
|
||||||
|
'package_fullname': '_'.join([raw_tasks['package_name'],
|
||||||
|
raw_tasks['package_version'],
|
||||||
|
raw_tasks['package_release']]),
|
||||||
|
'started_at': started_at,
|
||||||
|
'step_stats': step_stats
|
||||||
|
}
|
||||||
|
result.append(TestTask(**params))
|
||||||
|
return result
|
||||||
|
|
||||||
|
def __parse_test_steps_stats(self, stats_raw: Dict[str, Any], results_raw: Dict[str, Any]) -> TestStepsStats:
|
||||||
|
teast_steps_params = {}
|
||||||
|
for field_name in TestStepsStats.__fields__.keys():
|
||||||
|
try:
|
||||||
|
p = stats_raw[field_name]
|
||||||
|
except KeyError:
|
||||||
|
continue
|
||||||
|
p['success'] = results_raw[field_name]['success']
|
||||||
|
teast_steps_params[field_name] = TestStepStat(**p)
|
||||||
|
return TestStepsStats(**teast_steps_params)
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
|
|
||||||
# supported schema version
|
# supported schema version
|
||||||
DB_SCHEMA_VER = 2
|
DB_SCHEMA_VER = 3
|
||||||
|
|
||||||
|
|
||||||
# ENUMS
|
# ENUMS
|
||||||
@ -41,3 +41,20 @@ class BuildNodeStatsEnum(IntEnum):
|
|||||||
build_node_task = 6
|
build_node_task = 6
|
||||||
cas_notarize_artifacts = 7
|
cas_notarize_artifacts = 7
|
||||||
cas_source_authenticate = 8
|
cas_source_authenticate = 8
|
||||||
|
|
||||||
|
|
||||||
|
class TestTaskStatusEnum(IntEnum):
|
||||||
|
created = 1
|
||||||
|
started = 2
|
||||||
|
completed = 3
|
||||||
|
failed = 4
|
||||||
|
|
||||||
|
|
||||||
|
class TestStepEnum(IntEnum):
|
||||||
|
install_package = 0
|
||||||
|
stop_enviroment = 1
|
||||||
|
initial_provision = 2
|
||||||
|
start_enviroment = 3
|
||||||
|
uninstall_package = 4
|
||||||
|
initialize_terraform = 5
|
||||||
|
package_integrity_tests = 6
|
||||||
|
@ -9,6 +9,7 @@ from .models.build_task_db import BuildTaskDB
|
|||||||
from .models.build_node_stat_db import BuildNodeStatDB
|
from .models.build_node_stat_db import BuildNodeStatDB
|
||||||
from .models.db_config import DbConfig
|
from .models.db_config import DbConfig
|
||||||
from .models.web_node_stat_db import WebNodeStatDB
|
from .models.web_node_stat_db import WebNodeStatDB
|
||||||
|
from .models.test_task_db import TestTaskDB
|
||||||
|
|
||||||
|
|
||||||
class DB():
|
class DB():
|
||||||
@ -229,3 +230,26 @@ class DB():
|
|||||||
cur.execute(sql, (build_task_id, stat_name_id))
|
cur.execute(sql, (build_task_id, stat_name_id))
|
||||||
val = int(cur.fetchone()[0])
|
val = int(cur.fetchone()[0])
|
||||||
return val == 1
|
return val == 1
|
||||||
|
|
||||||
|
def insert_test_task(self, task: TestTaskDB):
|
||||||
|
# inserting test task itself
|
||||||
|
sql = '''
|
||||||
|
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
|
||||||
|
VALUES (%s, %s, %s, %s, %s, %s);
|
||||||
|
'''
|
||||||
|
|
||||||
|
cur = self.__conn.cursor()
|
||||||
|
cur.execute(sql, (task.id, task.build_task_id, task.status_id,
|
||||||
|
task.package_fullname, task.started_at))
|
||||||
|
# inserting test steps stats
|
||||||
|
for ss in task.steps_stats:
|
||||||
|
sql = '''
|
||||||
|
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
|
||||||
|
VALUES
|
||||||
|
(%s, %s, %s, %s, %s);
|
||||||
|
'''
|
||||||
|
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
|
||||||
|
ss.start_ts, ss.end_ts, ss.success))
|
||||||
|
|
||||||
|
# commiting changes
|
||||||
|
self.__conn.commit()
|
||||||
|
@ -1,13 +1,13 @@
|
|||||||
# pylint: disable=relative-beyond-top-level
|
# pylint: disable=relative-beyond-top-level
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Dict
|
from typing import Dict, List
|
||||||
|
|
||||||
from ..models.extractor_config import ExtractorConfig
|
|
||||||
from ..const import BuildTaskEnum
|
|
||||||
from ..models.build import BuildTask
|
|
||||||
from ..db import DB
|
|
||||||
from ..api_client import APIclient
|
from ..api_client import APIclient
|
||||||
|
from ..const import BuildTaskEnum
|
||||||
|
from ..db import DB
|
||||||
|
from ..models.build import BuildTask
|
||||||
|
from ..models.extractor_config import ExtractorConfig
|
||||||
|
|
||||||
|
|
||||||
class Extractor:
|
class Extractor:
|
||||||
@ -36,7 +36,7 @@ class Extractor:
|
|||||||
break
|
break
|
||||||
|
|
||||||
# inserting build build tasks and build tasks statistics
|
# inserting build build tasks and build tasks statistics
|
||||||
logging.info("inserting %s", build.id)
|
logging.info('inserting %s', build.id)
|
||||||
try:
|
try:
|
||||||
self.db.insert_build(build.as_db_model())
|
self.db.insert_build(build.as_db_model())
|
||||||
except Exception as error: # pylint: disable=broad-except
|
except Exception as error: # pylint: disable=broad-except
|
||||||
@ -45,6 +45,8 @@ class Extractor:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
for build_task in build.build_tasks:
|
for build_task in build.build_tasks:
|
||||||
|
logging.info('build %s: inserting build task %s',
|
||||||
|
build.id, build_task.id)
|
||||||
try:
|
try:
|
||||||
self.db.insert_buildtask(build_task.as_db_model(),
|
self.db.insert_buildtask(build_task.as_db_model(),
|
||||||
build_task.web_node_stats.as_db_model(
|
build_task.web_node_stats.as_db_model(
|
||||||
@ -52,8 +54,8 @@ class Extractor:
|
|||||||
build_task.build_node_stats.as_db_model(
|
build_task.build_node_stats.as_db_model(
|
||||||
build_task.id))
|
build_task.id))
|
||||||
except Exception as error: # pylint: disable=broad-except
|
except Exception as error: # pylint: disable=broad-except
|
||||||
logging.error('failed to insert build task %d: %s',
|
logging.error('build %s: failed to insert build task %d: %s',
|
||||||
build_task.id, error, exc_info=True)
|
build.id, build_task.id, error, exc_info=True)
|
||||||
build_count += 1
|
build_count += 1
|
||||||
page_num += 1
|
page_num += 1
|
||||||
return build_count
|
return build_count
|
||||||
|
10
build_analytics/build_analytics/models/test_step_stat.py
Normal file
10
build_analytics/build_analytics/models/test_step_stat.py
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
||||||
|
|
||||||
|
|
||||||
|
class TestStepStat(BaseModel):
|
||||||
|
start_ts: Optional[datetime] = None
|
||||||
|
end_ts: Optional[datetime] = None
|
||||||
|
success: bool
|
10
build_analytics/build_analytics/models/test_step_stat_db.py
Normal file
10
build_analytics/build_analytics/models/test_step_stat_db.py
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class TestStepStatDB(BaseModel):
|
||||||
|
test_task_id: int
|
||||||
|
stat_name_id: int
|
||||||
|
start_ts: Optional[float] = None
|
||||||
|
end_ts: Optional[float] = None
|
||||||
|
success: bool
|
37
build_analytics/build_analytics/models/test_steps_stats.py
Normal file
37
build_analytics/build_analytics/models/test_steps_stats.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
||||||
|
|
||||||
|
from ..const import TestStepEnum
|
||||||
|
from .test_step_stat import TestStepStat
|
||||||
|
from .test_step_stat_db import TestStepStatDB
|
||||||
|
|
||||||
|
|
||||||
|
class TestStepsStats(BaseModel):
|
||||||
|
install_package: Optional[TestStepStat] = None
|
||||||
|
stop_environment: Optional[TestStepStat] = None
|
||||||
|
initial_provision: Optional[TestStepStat] = None
|
||||||
|
start_environment: Optional[TestStepStat] = None
|
||||||
|
uninstall_package: Optional[TestStepStat] = None
|
||||||
|
initialize_terraform: Optional[TestStepStat] = None
|
||||||
|
package_integrity_tests: Optional[TestStepStat] = None
|
||||||
|
|
||||||
|
def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
|
||||||
|
result = []
|
||||||
|
for field_name in self.__fields__.keys():
|
||||||
|
stats: TestStepStat = getattr(self, field_name)
|
||||||
|
if not stats:
|
||||||
|
continue
|
||||||
|
start_ts = stats.start_ts.timestamp() \
|
||||||
|
if stats.start_ts else None
|
||||||
|
end_ts = stats.end_ts.timestamp() \
|
||||||
|
if stats.end_ts else None
|
||||||
|
stat_name_id = TestStepEnum[field_name].value
|
||||||
|
|
||||||
|
test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
|
||||||
|
stat_name_id=stat_name_id,
|
||||||
|
start_ts=start_ts,
|
||||||
|
end_ts=end_ts,
|
||||||
|
success=stats.success)
|
||||||
|
result.append(test_step_stat_db)
|
||||||
|
return result
|
31
build_analytics/build_analytics/models/test_task.py
Normal file
31
build_analytics/build_analytics/models/test_task.py
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
||||||
|
|
||||||
|
from .test_task_db import TestTaskDB
|
||||||
|
from .test_steps_stats import TestStepsStats
|
||||||
|
|
||||||
|
|
||||||
|
class TestTask(BaseModel):
|
||||||
|
id: int
|
||||||
|
build_task_id: int
|
||||||
|
revision: int
|
||||||
|
status: int
|
||||||
|
package_fullname: str
|
||||||
|
started_at: Optional[datetime] = None
|
||||||
|
steps_stats: TestStepsStats
|
||||||
|
|
||||||
|
def as_db_model(self) -> TestTaskDB:
|
||||||
|
started_at = self.started_at.timestamp() \
|
||||||
|
if self.started_at else None
|
||||||
|
params = {
|
||||||
|
'id': self.id,
|
||||||
|
'build_task_id': self.build_task_id,
|
||||||
|
'revision': self.revision,
|
||||||
|
'status': self.status,
|
||||||
|
'package_fullname': self.package_fullname,
|
||||||
|
'started_at': started_at,
|
||||||
|
'steps_stats': self.step_stats.as_db(self.id)
|
||||||
|
}
|
||||||
|
return TestTaskDB(**params)
|
17
build_analytics/build_analytics/models/test_task_db.py
Normal file
17
build_analytics/build_analytics/models/test_task_db.py
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
||||||
|
|
||||||
|
from .test_step_stat_db import TestStepStatDB
|
||||||
|
|
||||||
|
|
||||||
|
class TestTaskDB(BaseModel):
|
||||||
|
"""
|
||||||
|
Test task as it received from/sent to database
|
||||||
|
"""
|
||||||
|
id: int
|
||||||
|
build_task_id: int
|
||||||
|
revision: int
|
||||||
|
status_id: int
|
||||||
|
package_fullname: str
|
||||||
|
started_at: float
|
||||||
|
steps_stats: List[TestStepStatDB] = None
|
@ -1,75 +1,74 @@
|
|||||||
BEGIN;
|
BEGIN;
|
||||||
|
|
||||||
-- test_task_status_enum
|
-- test_tasks_status_enum
|
||||||
CREATE TABLE test_task_status_enum(
|
CREATE TABLE test_tasks_status_enum(
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
value VARCHAR(15)
|
value VARCHAR(15)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
INSERT INTO test_tasks_status_enum (id, value)
|
||||||
|
|
||||||
|
|
||||||
INSERT INTO test_task_status_enum (id, value)
|
|
||||||
VALUES
|
VALUES
|
||||||
(0, 'created'),
|
(1, 'created'),
|
||||||
(1, 'started'),
|
(2, 'started'),
|
||||||
(2, 'completed'),
|
(3, 'completed'),
|
||||||
(3, 'failed');
|
(4, 'failed');
|
||||||
|
|
||||||
|
|
||||||
-- test_task
|
-- test_tasks
|
||||||
CREATE TABLE test_task (
|
CREATE TABLE test_tasks (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
|
build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
|
||||||
revision INTEGER,
|
revision INTEGER,
|
||||||
status_id INTEGER REFERENCES test_task_status_enum(id) ON DELETE SET NULL,
|
status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
|
||||||
package_fullname VARCHAR(100)
|
package_fullname VARCHAR(100),
|
||||||
|
started_at DOUBLE PRECISION
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX test_task_build_task_id
|
CREATE INDEX test_tasks_build_task_id
|
||||||
ON test_task(build_task_id);
|
ON test_tasks(build_task_id);
|
||||||
|
|
||||||
CREATE INDEX test_task_build_status_id
|
CREATE INDEX test_tasks_build_status_id
|
||||||
ON test_task(status_id);
|
ON test_tasks(status_id);
|
||||||
|
|
||||||
CREATE INDEX test_task_package_fullname
|
CREATE INDEX test_tasks_package_fullname
|
||||||
ON test_task(package_fullname);
|
ON test_tasks(package_fullname);
|
||||||
|
|
||||||
|
|
||||||
-- test_step_enum
|
-- test_steps_enum
|
||||||
CREATE TABLE test_step_enum (
|
CREATE TABLE test_steps_enum (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
value VARCHAR(50)
|
value VARCHAR(50)
|
||||||
);
|
);
|
||||||
|
|
||||||
INSERT INTO test_step_enum (id, value)
|
INSERT INTO test_steps_enum (id, value)
|
||||||
VALUES
|
VALUES
|
||||||
(1, 'install_package'),
|
(0, 'install_package'),
|
||||||
(2, 'stop_environment'),
|
(1, 'stop_environment'),
|
||||||
(3, 'initial_provision'),
|
(2, 'initial_provision'),
|
||||||
(4, 'start_environment'),
|
(3, 'start_environment'),
|
||||||
(5, 'uninstall_package'),
|
(4, 'uninstall_package'),
|
||||||
(6, 'initialize_terraform'),
|
(5, 'initialize_terraform'),
|
||||||
(7, 'package_integrity_tests');
|
(6, 'package_integrity_tests');
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
-- test_step
|
-- test_steps
|
||||||
CREATE TABLE test_step_stats(
|
CREATE TABLE test_steps_stats(
|
||||||
test_task_id INTEGER,
|
test_task_id INTEGER,
|
||||||
stat_name_id INTEGER REFERENCES test_step_enum(id) ON DELETE SET NULL,
|
stat_name_id INTEGER REFERENCES (id) ON DELETE SET NULL,
|
||||||
start_ts DOUBLE PRECISION,
|
start_ts DOUBLE PRECISION,
|
||||||
end_ts DOUBLE PRECISION
|
end_ts DOUBLE PRECISION,
|
||||||
|
success BOOLEAN
|
||||||
);
|
);
|
||||||
|
|
||||||
ALTER TABLE test_step_stats
|
ALTER TABLE test_steps_stats
|
||||||
ADD CONSTRAINT test_step_stats_unique UNIQUE (test_task_id, stat_name_id);
|
ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
|
||||||
|
|
||||||
CREATE INDEX test_step_stats_start_ts
|
CREATE INDEX test_steps_stats_start_ts
|
||||||
ON test_step_stats(start_ts);
|
ON test_steps_stats(start_ts);
|
||||||
|
|
||||||
CREATE INDEX test_step_stats_end_ts
|
CREATE INDEX test_steps_stats_end_ts
|
||||||
ON test_step_stats(end_ts);
|
ON test_steps_stats(end_ts);
|
||||||
|
|
||||||
|
|
||||||
UPDATE schema_version
|
UPDATE schema_version
|
||||||
|
Loading…
Reference in New Issue
Block a user