build_analytics/build_analytics/api_client.py

@@ -47,18 +47,11 @@ class APIclient():
                               b, err, exc_info=True)
         return result
 
-    def get_build(self, build_id: int) -> Optional[Build]:
-        '''
-        method returns None if build was deleted from ALBS
-        '''
+    def get_build(self, build_id: int) -> Build:
         ep = f'/api/v1/builds/{build_id}'
         url = urljoin(self.api_root, ep)
         headers = {'accept': 'application/json'}
         response = requests.get(url, headers=headers, timeout=self.timeout)
-
-        if response.status_code == 404:
-            return None
-
         response.raise_for_status()
         return self._parse_build(response.json())
 

build_analytics/build_analytics/db.py

@@ -62,34 +62,34 @@ class DB():
                           build_task.started_at, build_task.finished_at, build_task.status_id))
 
         # inserting web node stats
-        for wn_stat in web_node_stats:
+        for stat in web_node_stats:
 
             # do not insert empty stats
-            if wn_stat.start_ts is None:
+            if stat.start_ts is None:
                 continue
 
             sql = '''
                     INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
                 '''
-            cur.execute(sql, (wn_stat.build_task_id, wn_stat.stat_name_id,
-                        wn_stat.start_ts, wn_stat.end_ts))
+            cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
+                        stat.start_ts, stat.end_ts))
             logging.debug('raw SQL query: %s', cur.query)
             self.__conn.commit()
 
         # inserting build node stats
-        for bn_stat in build_node_stats:
+        for stat in build_node_stats:
 
             # do not insert empty stats
-            if bn_stat.start_ts is None:
+            if stat.start_ts is None:
                 continue
 
             sql = '''
                     INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
                     VALUES (%s, %s, %s, %s);
                 '''
-            cur.execute(sql, (bn_stat.build_task_id, bn_stat.stat_name_id,
-                        bn_stat.start_ts, bn_stat.end_ts))
+            cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
+                        stat.start_ts, stat.end_ts))
             logging.debug('raw SQL query: %s', cur.query)
 
         # commiting changes
@@ -121,12 +121,11 @@ class DB():
 
         # getting unfinished builds
         sql = 'SELECT id FROM builds where finished_at is NULL AND created_at > %s;'
-        builds_to_check: Dict[int, bool] = {}
         cur = self.__conn.cursor()
         cur.execute(sql, (not_before.timestamp(),))
         logging.debug('raw SQL query: %s', cur.query)
         for row in cur.fetchall():
-            builds_to_check[row[0]] = True
+            res[row[0]] = {}
 
         # getting list of unfinished tasks
         sql = 'SELECT id, build_id, status_id FROM build_tasks WHERE status_id < 2;'
@@ -136,8 +135,6 @@ class DB():
             build_task_id: int = row[0]
             build_id: int = row[1]
             status_id: int = row[2]
-            if build_id not in builds_to_check:
-                continue
             try:
                 res[build_id][build_task_id] = status_id
             except KeyError:
@@ -198,11 +195,11 @@ class DB():
             logging.debug('raw SQL query: %s', cur.query)
 
         # updating build_node_stats
-        for bn_stat in build_node_stats:
+        for stat in build_node_stats:
             logging.debug(
-                'updating build_node_stats %s build_task %s', bn_stat.stat_name_id, build_task.id)
-            if self.stat_exists(task_id=bn_stat.build_task_id,
-                                stat_name_id=bn_stat.stat_name_id,
+                'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
+            if self.stat_exists(task_id=stat.build_task_id,
+                                stat_name_id=stat.stat_name_id,
                                 table_name='build_node_stats',
                                 column_name='build_task_id'):
                 sql = '''
@@ -216,9 +213,9 @@ class DB():
                         VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
                       '''
             params = {'build_task_id': build_task.id,
-                      'stat_name_id': bn_stat.stat_name_id,
-                      'start_ts': bn_stat.start_ts,
-                      'end_ts': bn_stat.end_ts}
+                      'stat_name_id': stat.stat_name_id,
+                      'start_ts': stat.start_ts,
+                      'end_ts': stat.end_ts}
             logging.debug('raw SQL query: %s', cur.query)
             cur.execute(sql, params)
 
@@ -321,11 +318,3 @@ class DB():
                                 s.start_ts, s.finish_ts))
         # commiting changes
         self.__conn.commit()
-
-    def delete_build(self, build_id: int):
-        params = (build_id,)
-        sql = "DELETE FROM builds WHERE id = %s;"
-        cur = self.__conn.cursor()
-
-        cur.execute(sql, params)
-        self.__conn.commit()

build_analytics/build_analytics/extractor/extractor.py

@@ -105,19 +105,13 @@ class Extractor:
                     b.build_id, b.id, BuildTaskEnum(b.status_id).name)
 
     def update_builds(self):
-        logging.info('Getting unfinished builds that were created after %s ',
-                     self.config.oldest_to_update)
+        logging.info('Getting list of tasks from DB')
         unfinished_tasks = self.db.get_unfinished_builds(
             self.config.oldest_to_update)
         for build_id, build_tasks_db in unfinished_tasks.items():
             try:
                 logging.info('Getting status of build %d', build_id)
                 build = self.api.get_build(build_id)
-                if not build:
-                    logging.warning(
-                        "build %s was deleted from albs, removing it", build_id)
-                    self.db.delete_build(build_id)
-                    continue
 
                 logging.info('Updating build tasks')
                 build_tasks_to_check = [

build_analytics/build_analytics/extractor/start.py

@@ -3,7 +3,6 @@ import logging
 from logging.handlers import RotatingFileHandler
 import sys
 import time
-from typing import Dict, Any
 
 import yaml
 
@@ -27,15 +26,11 @@ def __get_config(yml_path: str) -> ExtractorConfig:
     raw['oldest_build_age'] = datetime.now().astimezone() \
         - timedelta(days=raw['data_store_days'])
 
-    # Dbconfig
-    db_params: Dict[str, Any] = {'name': raw['db_name'],
-                                 'username': raw['db_username'],
-                                 'password': raw['db_password'], }
-    if 'db_port' in raw:
-        db_params['port'] = raw['db_port']
-    if 'db_host' in raw:
-        db_params['host'] = raw['db_host']
-    raw['db_config'] = DbConfig(**db_params)
+    raw['db_config'] = DbConfig(name=raw['db_name'],
+                                port=int(raw['db_port']),
+                                host=raw['db_host'],
+                                username=raw['db_username'],
+                                password=raw['db_password'])
 
     if 'oldest_to_update_days' in raw:
         raw['oldest_to_update_days'] = datetime.now().astimezone() \

build_analytics/build_analytics/models/db_config.py

@@ -1,13 +1,9 @@
 from pydantic import BaseModel, Field
 
 
-DB_PORT = 5432
-DB_HOST = "localhost"
-
-
 class DbConfig(BaseModel):
     name: str = Field(description="db name")
-    port: int = Field(description="db server port", default=DB_PORT)
-    host: str = Field(description="db server ip/hostname", default=DB_HOST)
+    port: int = Field(description="db server port")
+    host: str = Field(description="db server ip/hostname")
     username: str = Field(description="username to connect with")
     password: str = Field(description="password to connect with1")

releases.txt

@@ -21,12 +21,7 @@ First version
 0.3.2 (2023-03-23)
  - Bugfix ALBS-1060
 
-0.3.3 (2023-04-24)
-build-analytics
-  Improvements
-    - [ALBS-1077] start deleting builds that were removed from ALBS
-  Bugfixes
-    - 'Key error' when db_port/db_host is not set
-    - update_builds() ignoring opldest_to_update attribute
-    - [ALBS-1099] Test task started_at attribute is NULL
-    - Max recursion error in 'Test task details.json'
+0.3.3 (IN PROGRESS)
+build-analytics:
+ - [ALBS-1099] change source of Test task started_at timestamp
+ - [ALBS-1077] start deleting builds that were removed from ALBS