build_analytics/build_analytics/api_client.py

@@ -1,19 +1,16 @@
 from datetime import datetime
 import logging
 from urllib.parse import urljoin
-from typing import Dict, List, Any
+from typing import Dict, List
 
 import requests
 
-
 from .models.build import Build
 from .models.build_task import BuildTask
 from .models.build_node_stats import BuildNodeStats
 from .models.build_stat import BuildStat
 from .models.web_node_stats import WebNodeStats
-from .models.test_task import TestTask
+
-from .models.test_steps_stats import TestStepsStats
-from .models.test_step_stat import TestStepStat
 
 TZ_OFFSET = '+00:00'
 
@@ -141,50 +138,3 @@ class APIclient():
             'build_tasks': build_tasks}
 
         return Build(**params)
-
-    def get_test_tasks(self, build_task_id: int) -> List[TestTask]:
-        ep = f'/api/v1/tests/{build_task_id}/latest'
-        url = urljoin(self.api_root, ep)
-        headers = {'accept': 'application/json'}
-
-        response = requests.get(
-            url, headers=headers, timeout=self.timeout)
-        response.raise_for_status()
-        return self.__parse_test_tasks(response.json(), build_task_id)
-
-    def __parse_test_tasks(self, raw_tasks: List[Dict[str, Any]],
-                           build_task_id: int,
-                           started_at: str = None) -> List[TestTask]:
-        result: List[TestTask] = []
-        for task in raw_tasks:
-            if task['alts_response']:
-                started_raw = task['alts_response']['stats']['started_at']
-                started_at = datetime.fromisoformat(started_raw+TZ_OFFSET)
-                stats_raw = task['alts_response']['stats']
-                steps_stats = self.__parse_test_steps_stats(stats_raw)
-            else:
-                started_at = None
-                steps_stats = None
-            params = {
-                'id': task['id'],
-                'build_task_id': build_task_id,
-                'revision': task['revision'],
-                'status': task['status'],
-                'package_fullname': '_'.join([task['package_name'],
-                                              task['package_version'],
-                                              task['package_release']]),
-                'started_at': started_at,
-                'steps_stats': steps_stats
-            }
-            result.append(TestTask(**params))
-        return result
-
-    def __parse_test_steps_stats(self, stats_raw: Dict[str, Any]) -> TestStepsStats:
-        teast_steps_params = {}
-        for field_name in TestStepsStats.__fields__.keys():
-            try:
-                p = stats_raw[field_name]
-            except KeyError:
-                continue
-            teast_steps_params[field_name] = TestStepStat(**p)
-        return TestStepsStats(**teast_steps_params)

build_analytics/build_analytics/const.py

@@ -3,7 +3,7 @@
 from enum import IntEnum
 
 # supported schema version
-DB_SCHEMA_VER = 3
+DB_SCHEMA_VER = 2
 
 
 # ENUMS
@@ -41,21 +41,3 @@ class BuildNodeStatsEnum(IntEnum):
     build_node_task = 6
     cas_notarize_artifacts = 7
     cas_source_authenticate = 8
-
-
-class TestTaskStatusEnum(IntEnum):
-    created = 1
-    started = 2
-    completed = 3
-    failed = 4
-
-
-class TestStepEnum(IntEnum):
-    install_package = 0
-    stop_enviroment = 1
-    initial_provision = 2
-    start_environment = 3
-    uninstall_package = 4
-    initialize_terraform = 5
-    package_integrity_tests = 6
-    stop_environment = 7

build_analytics/build_analytics/db.py

@@ -9,7 +9,6 @@ from .models.build_task_db import BuildTaskDB
 from .models.build_node_stat_db import BuildNodeStatDB
 from .models.db_config import DbConfig
 from .models.web_node_stat_db import WebNodeStatDB
-from .models.test_task_db import TestTaskDB
 
 
 class DB():
@@ -230,28 +229,3 @@ class DB():
         cur.execute(sql, (build_task_id, stat_name_id))
         val = int(cur.fetchone()[0])
         return val == 1
-
-    def insert_test_task(self, task: TestTaskDB):
-        cur = self.__conn.cursor()
-        # inserting test task itself
-        sql = '''
-                INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
-                VALUES
-                  (%s, %s, %s, %s, %s, %s);
-              '''
-        cur.execute(sql, (task.id, task.build_task_id, task.revision, task.status_id,
-                          task.package_fullname, task.started_at))
-
-        if task.steps_stats:
-            # inserting  test steps stats
-            for ss in task.steps_stats:
-                sql = '''
-                    INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, finish_ts)
-                    VALUES
-                        (%s, %s, %s, %s);
-                    '''
-                cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
-                            ss.start_ts, ss.finish_ts))
-
-        # commiting changes
-        self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -1,13 +1,13 @@
 # pylint: disable=relative-beyond-top-level
 
 import logging
-from typing import Dict, List
+from typing import List, Dict
 
-from ..api_client import APIclient
-from ..const import BuildTaskEnum
-from ..db import DB
-from ..models.build import BuildTask
 from ..models.extractor_config import ExtractorConfig
+from ..const import BuildTaskEnum
+from ..models.build import BuildTask
+from ..db import DB
+from ..api_client import APIclient
 
 
 class Extractor:
@@ -36,7 +36,7 @@ class Extractor:
                     break
 
                 # inserting build build tasks and build tasks statistics
-                logging.info('inserting %s', build.id)
+                logging.info("inserting %s", build.id)
                 try:
                     self.db.insert_build(build.as_db_model())
                 except Exception as error:  # pylint: disable=broad-except
@@ -45,8 +45,6 @@ class Extractor:
                     continue
 
                 for build_task in build.build_tasks:
-                    logging.info('build %s: inserting build task %s',
-                                 build.id, build_task.id)
                     try:
                         self.db.insert_buildtask(build_task.as_db_model(),
                                                  build_task.web_node_stats.as_db_model(
@@ -54,17 +52,8 @@ class Extractor:
                                                  build_task.build_node_stats.as_db_model(
                             build_task.id))
                     except Exception as error:  # pylint: disable=broad-except
-                        logging.error('build %s: failed to insert build task %d: %s',
+                        logging.error('failed to insert build task %d: %s',
-                                      build.id, build_task.id, error, exc_info=True)
+                                      build_task.id, error, exc_info=True)
-
-                    logging.info(
-                        'getting test tasks for build task %s', build_task.id)
-                    test_tasks = self.api.get_test_tasks(build_task.id)
-                    logging.info('received %d tests tasks', len(test_tasks))
-                    for t in test_tasks:
-                        logging.info(
-                            'build task %s: inserting test task %s', build_task.id, t.id)
-                        self.db.insert_test_task(t.as_db_model())
                 build_count += 1
             page_num += 1
         return build_count

build_analytics/build_analytics/models/sign_task_db.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
+from pydantic import BaseModel
 
 
 class SignTaskDB(BaseModel):

build_analytics/build_analytics/models/test_step_stat.py

@@ -1,9 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
-
-
-class TestStepStat(BaseModel):
-    start_ts: Optional[datetime] = None
-    finish_ts: Optional[datetime] = None

build_analytics/build_analytics/models/test_step_stat_db.py

@@ -1,9 +0,0 @@
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
-from typing import Optional
-
-
-class TestStepStatDB(BaseModel):
-    test_task_id: int
-    stat_name_id: int
-    start_ts: Optional[float] = None
-    finish_ts: Optional[float] = None

build_analytics/build_analytics/models/test_steps_stats.py

@@ -1,36 +0,0 @@
-from typing import List, Optional
-
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
-
-from ..const import TestStepEnum
-from .test_step_stat import TestStepStat
-from .test_step_stat_db import TestStepStatDB
-
-
-class TestStepsStats(BaseModel):
-    install_package: Optional[TestStepStat] = None
-    stop_environment: Optional[TestStepStat] = None
-    initial_provision: Optional[TestStepStat] = None
-    start_environment: Optional[TestStepStat] = None
-    uninstall_package: Optional[TestStepStat] = None
-    initialize_terraform: Optional[TestStepStat] = None
-    package_integrity_tests: Optional[TestStepStat] = None
-
-    def as_db(self, test_task_id: int) -> List[TestStepStatDB]:
-        result = []
-        for field_name in self.__fields__.keys():
-            stats: TestStepStat = getattr(self, field_name)
-            if not stats:
-                continue
-            start_ts = stats.start_ts.timestamp() \
-                if stats.start_ts else None
-            finish_ts = stats.finish_ts.timestamp() \
-                if stats.finish_ts else None
-            stat_name_id = TestStepEnum[field_name].value
-
-            test_step_stat_db = TestStepStatDB(test_task_id=test_task_id,
-                                               stat_name_id=stat_name_id,
-                                               start_ts=start_ts,
-                                               finish_ts=finish_ts)
-            result.append(test_step_stat_db)
-        return result

build_analytics/build_analytics/models/test_task.py

@@ -1,31 +0,0 @@
-from datetime import datetime
-from typing import Optional
-
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
-
-from .test_task_db import TestTaskDB
-from .test_steps_stats import TestStepsStats
-
-
-class TestTask(BaseModel):
-    id: int
-    build_task_id: int
-    revision: int
-    status: int
-    package_fullname: str
-    started_at: Optional[datetime] = None
-    steps_stats: Optional[TestStepsStats] = None
-
-    def as_db_model(self) -> TestTaskDB:
-        started_at = self.started_at.timestamp() \
-            if self.started_at else None
-        params = {
-            'id': self.id,
-            'build_task_id': self.build_task_id,
-            'revision': self.revision,
-            'status_id': self.status,
-            'package_fullname': self.package_fullname,
-            'started_at': started_at,
-            'steps_stats': self.steps_stats.as_db(self.id) if self.steps_stats else None
-        }
-        return TestTaskDB(**params)

build_analytics/build_analytics/models/test_task_db.py

@@ -1,17 +0,0 @@
-from typing import List, Optional
-from pydantic import BaseModel  # pylint: disable=no-name-in-module
-
-from .test_step_stat_db import TestStepStatDB
-
-
-class TestTaskDB(BaseModel):
-    """
-    Test task as it received from/sent to database
-    """
-    id: int
-    build_task_id: int
-    revision: int
-    status_id: int
-    package_fullname: str
-    started_at: Optional[float] = None
-    steps_stats: Optional[List[TestStepStatDB]] = None

build_analytics/migrations/1.sql

@@ -16,7 +16,7 @@ CREATE INDEX IF NOT EXISTS builds_finished_at
 ON builds(finished_at);
 
 
--- build_tasks_enum
+-- build_taks_enum
 CREATE TABLE IF NOT EXISTS build_task_status_enum(
     id INTEGER PRIMARY KEY,
     value VARCHAR(15)
@@ -53,7 +53,7 @@ CREATE TABLE web_node_stats_enum (
 );
 
 INSERT INTO web_node_stats_enum (id, value)
-VALUES
+VALUEs
     (0, 'build_done'),
     (1, 'logs_processing'),
     (2, 'packages_processing');

build_analytics/migrations/3.sql

@@ -1,77 +0,0 @@
-BEGIN;
-
--- test_tasks_status_enum
-CREATE TABLE  test_tasks_status_enum(
-    id INTEGER PRIMARY KEY,
-    value VARCHAR(15)
-);
-
-INSERT INTO test_tasks_status_enum (id, value)
-VALUES
-    (1, 'created'),
-    (2, 'started'),
-    (3, 'completed'),
-    (4, 'failed');
-
-
--- test_tasks
-CREATE TABLE test_tasks (
-    id INTEGER PRIMARY KEY,
-    build_task_id INTEGER REFERENCES build_tasks(id) ON DELETE CASCADE,
-    revision INTEGER,
-    status_id INTEGER REFERENCES test_tasks_status_enum(id) ON DELETE SET NULL,
-    package_fullname VARCHAR(100),
-    started_at DOUBLE PRECISION
-);
-
-CREATE INDEX test_tasks_build_task_id
-ON test_tasks(build_task_id);
-
-CREATE INDEX test_tasks_build_status_id
-ON test_tasks(status_id);
-
-CREATE INDEX test_tasks_package_fullname
-ON test_tasks(package_fullname);
-
-
--- test_steps_enum
-CREATE TABLE test_steps_enum (
-    id INTEGER PRIMARY KEY,
-    value VARCHAR(50)
-);
-
-INSERT INTO test_steps_enum (id, value)
-VALUES
-    (0, 'install_package'),
-    (1, 'stop_environment'),
-    (2, 'initial_provision'),
-    (3, 'start_environment'),
-    (4, 'uninstall_package'),
-    (5, 'initialize_terraform'),
-    (6, 'package_integrity_tests'),
-    (7, 'stop_environment');
-
-
-
--- test_steps
-CREATE TABLE test_steps_stats(
-    test_task_id INTEGER REFERENCES test_tasks(id) ON DELETE CASCADE,
-    stat_name_id  INTEGER REFERENCES test_steps_enum(id) ON DELETE SET NULL,
-    start_ts DOUBLE PRECISION,
-    finish_ts DOUBLE PRECISION
-);
-
-ALTER TABLE test_steps_stats
-ADD CONSTRAINT test_steps_stats_unique UNIQUE (test_task_id, stat_name_id);
-
-CREATE INDEX test_steps_stats_start_ts
-ON test_steps_stats(start_ts);
-
-CREATE INDEX test_steps_stats_end_ts
-ON test_steps_stats(end_ts);
-
-
-UPDATE schema_version
-SET version = 3;
-
-COMMIT;

build_analytics/releases.txt

@@ -7,7 +7,4 @@ First version
 - Added metrics for build steps
 
 0.2.1 (2023-03-15)
- - Added canceled Build task status
+ - Added canceled Build task status
-
-0.3.0 (IN PROGRESS)
- - Added test tasks stats