2023-02-27 19:51:53 +00:00
|
|
|
from datetime import datetime
|
2023-03-07 15:02:14 +00:00
|
|
|
from typing import Union, Dict, List, Optional
|
|
|
|
import logging
|
2023-02-27 19:51:53 +00:00
|
|
|
|
|
|
|
import psycopg2
|
|
|
|
|
|
|
|
from .models.build_db import BuildDB
|
|
|
|
from .models.build_task_db import BuildTaskDB
|
2023-03-06 18:21:56 +00:00
|
|
|
from .models.build_node_stat_db import BuildNodeStatDB
|
2023-02-27 19:51:53 +00:00
|
|
|
from .models.db_config import DbConfig
|
2023-03-06 18:21:56 +00:00
|
|
|
from .models.web_node_stat_db import WebNodeStatDB
|
2023-03-15 10:28:12 +00:00
|
|
|
from .models.test_task_db import TestTaskDB
|
2023-02-27 19:51:53 +00:00
|
|
|
|
|
|
|
|
|
|
|
class DB():
|
|
|
|
def __init__(self, config: DbConfig):
|
2023-02-28 17:28:48 +00:00
|
|
|
self.__conn = psycopg2.connect(database=config.name,
|
|
|
|
host=config.host,
|
|
|
|
user=config.username,
|
|
|
|
password=config.password,
|
|
|
|
port=config.port)
|
|
|
|
|
|
|
|
def close_conn(self):
|
|
|
|
self.__conn.close()
|
|
|
|
|
|
|
|
def __del__(self):
|
|
|
|
self.close_conn()
|
|
|
|
|
|
|
|
def insert_build(self, build: BuildDB):
|
|
|
|
sql = '''
|
2023-02-27 19:51:53 +00:00
|
|
|
INSERT INTO builds(id, url, created_at, finished_at)
|
2023-02-28 17:28:48 +00:00
|
|
|
VALUES (%s, %s, %s, %s);
|
|
|
|
'''
|
|
|
|
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql, (build.id, build.url,
|
|
|
|
build.created_at, build.finished_at))
|
|
|
|
self.__conn.commit()
|
|
|
|
|
2023-03-06 18:21:56 +00:00
|
|
|
def insert_buildtask(self, build_task: BuildTaskDB, web_node_stats: List[WebNodeStatDB],
|
|
|
|
build_node_stats: List[BuildNodeStatDB]):
|
|
|
|
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
# inserting build_task
|
2023-02-28 17:28:48 +00:00
|
|
|
sql = '''
|
2023-03-01 13:13:14 +00:00
|
|
|
INSERT INTO build_tasks(id, name, build_id, arch_id, started_at, finished_at, status_id)
|
|
|
|
VALUES (%s, %s, %s, %s, %s, %s, %s);
|
2023-02-28 17:28:48 +00:00
|
|
|
'''
|
2023-03-01 13:13:14 +00:00
|
|
|
cur.execute(sql, (build_task.id, build_task.name, build_task.build_id, build_task.arch_id,
|
2023-02-28 17:28:48 +00:00
|
|
|
build_task.started_at, build_task.finished_at, build_task.status_id))
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# inserting web node stats
|
|
|
|
for stat in web_node_stats:
|
2023-03-10 18:35:12 +00:00
|
|
|
|
|
|
|
# do not insert empty stats
|
|
|
|
if stat.start_ts is None:
|
|
|
|
continue
|
|
|
|
|
2023-03-06 18:21:56 +00:00
|
|
|
sql = '''
|
|
|
|
INSERT INTO web_node_stats (build_task_id, stat_name_id, start_ts, end_ts)
|
|
|
|
VALUES (%s, %s, %s, %s);
|
|
|
|
'''
|
|
|
|
cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
|
|
|
|
stat.start_ts, stat.end_ts))
|
2023-03-07 15:02:14 +00:00
|
|
|
logging.debug('raw SQL query: %s', cur.query)
|
2023-03-10 18:35:12 +00:00
|
|
|
self.__conn.commit()
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# inserting build node stats
|
|
|
|
for stat in build_node_stats:
|
2023-03-10 18:35:12 +00:00
|
|
|
|
|
|
|
# do not insert empty stats
|
|
|
|
if stat.start_ts is None:
|
|
|
|
continue
|
|
|
|
|
2023-03-06 18:21:56 +00:00
|
|
|
sql = '''
|
|
|
|
INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
|
|
|
|
VALUES (%s, %s, %s, %s);
|
|
|
|
'''
|
|
|
|
cur.execute(sql, (stat.build_task_id, stat.stat_name_id,
|
|
|
|
stat.start_ts, stat.end_ts))
|
2023-03-07 15:02:14 +00:00
|
|
|
logging.debug('raw SQL query: %s', cur.query)
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# commiting changes
|
2023-02-28 17:28:48 +00:00
|
|
|
self.__conn.commit()
|
2023-02-27 19:51:53 +00:00
|
|
|
|
|
|
|
def get_latest_build_id(self) -> Union[int, None]:
|
|
|
|
sql = "SELECT id from builds ORDER BY id DESC LIMIT 1;"
|
2023-02-28 17:28:48 +00:00
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql)
|
|
|
|
val = cur.fetchone()
|
2023-02-27 19:51:53 +00:00
|
|
|
if not val:
|
|
|
|
return None
|
|
|
|
return int(val[0])
|
|
|
|
|
|
|
|
def cleanup_builds(self, oldest_to_keep: datetime) -> int:
|
|
|
|
params = (int(oldest_to_keep.timestamp()),)
|
|
|
|
sql = "DELETE FROM builds WHERE created_at < %s;"
|
2023-02-28 17:28:48 +00:00
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql, params)
|
|
|
|
self.__conn.commit()
|
2023-02-27 19:51:53 +00:00
|
|
|
return cur.rowcount
|
2023-02-28 17:28:48 +00:00
|
|
|
|
|
|
|
def get_unfinished_builds(self) -> Dict[int, Dict[int, int]]:
|
|
|
|
"""
|
|
|
|
Getting list of unfinished builds and build_tasks
|
|
|
|
Dict[build_id, Dict[build_task_id, task_status_id]]
|
|
|
|
"""
|
|
|
|
res: Dict[int, Dict[int, int]] = {}
|
|
|
|
|
|
|
|
# getting unfinished builds
|
|
|
|
sql = 'SELECT id FROM builds where finished_at is NULL;'
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql)
|
|
|
|
for row in cur.fetchall():
|
|
|
|
res[row[0]] = {}
|
|
|
|
|
|
|
|
# getting list of unfinished tasks
|
|
|
|
sql = 'SELECT id, build_id, status_id FROM build_tasks WHERE status_id < 2;'
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql)
|
|
|
|
for row in cur.fetchall():
|
|
|
|
build_task_id: int = row[0]
|
|
|
|
build_id: int = row[1]
|
|
|
|
status_id: int = row[2]
|
|
|
|
try:
|
|
|
|
res[build_id][build_task_id] = status_id
|
|
|
|
except KeyError:
|
|
|
|
res[build_id] = {build_task_id: status_id}
|
|
|
|
|
|
|
|
return res
|
|
|
|
|
|
|
|
def update_build(self, build: BuildDB):
|
|
|
|
sql = '''
|
|
|
|
UPDATE builds
|
|
|
|
SET finished_at = %s
|
|
|
|
WHERE id = %s;
|
|
|
|
'''
|
|
|
|
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql, (build.finished_at, build.id))
|
|
|
|
self.__conn.commit()
|
|
|
|
|
2023-03-06 18:21:56 +00:00
|
|
|
def update_build_task(self, build_task: BuildTaskDB,
|
|
|
|
web_node_stats: List[WebNodeStatDB],
|
|
|
|
build_node_stats: List[BuildNodeStatDB]):
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
|
2023-02-28 17:28:48 +00:00
|
|
|
sql = '''
|
|
|
|
UPDATE build_tasks
|
|
|
|
SET status_id = %s,
|
|
|
|
started_at = %s,
|
|
|
|
finished_at = %s
|
|
|
|
WHERE id = %s;
|
|
|
|
'''
|
2023-03-06 18:21:56 +00:00
|
|
|
cur.execute(sql, (build_task.status_id, build_task.started_at,
|
|
|
|
build_task.finished_at, build_task.id))
|
2023-03-10 18:35:12 +00:00
|
|
|
logging.debug('raw SQL query: %s', cur.query)
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# updating web_node_stats
|
|
|
|
for stat in web_node_stats:
|
2023-03-10 18:35:12 +00:00
|
|
|
logging.debug(
|
|
|
|
'updating web_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
|
|
|
|
if self.stat_exists(build_task_id=stat.build_task_id,
|
|
|
|
stat_name_id=stat.stat_name_id,
|
|
|
|
table_name='web_node_stats'):
|
|
|
|
sql = '''
|
|
|
|
UPDATE web_node_stats
|
|
|
|
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
|
|
|
|
WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
|
|
|
|
'''
|
|
|
|
else:
|
|
|
|
sql = '''
|
|
|
|
INSERT INTO web_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
|
|
|
|
VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
|
|
|
|
'''
|
|
|
|
params = {'build_task_id': build_task.id,
|
|
|
|
'stat_name_id': stat.stat_name_id,
|
|
|
|
'start_ts': stat.start_ts,
|
|
|
|
'end_ts': stat.end_ts}
|
|
|
|
cur.execute(sql, params)
|
|
|
|
logging.debug('raw SQL query: %s', cur.query)
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# updating build_node_stats
|
|
|
|
for stat in build_node_stats:
|
2023-03-10 18:35:12 +00:00
|
|
|
logging.debug(
|
|
|
|
'updating build_node_stats %s build_task %s', stat.stat_name_id, build_task.id)
|
|
|
|
if self.stat_exists(build_task_id=stat.build_task_id,
|
|
|
|
stat_name_id=stat.stat_name_id,
|
|
|
|
table_name='build_node_stats'):
|
|
|
|
sql = '''
|
|
|
|
UPDATE build_node_stats
|
|
|
|
SET start_ts = %(start_ts)s, end_ts = %(end_ts)s
|
|
|
|
WHERE build_task_id = %(build_task_id)s AND stat_name_id = %(stat_name_id)s
|
|
|
|
'''
|
|
|
|
else:
|
|
|
|
sql = '''
|
|
|
|
INSERT INTO build_node_stats(build_task_id, stat_name_id, start_ts, end_ts)
|
|
|
|
VALUES (%(build_task_id)s, %(stat_name_id)s, %(start_ts)s, %(end_ts)s);
|
|
|
|
'''
|
|
|
|
params = {'build_task_id': build_task.id,
|
|
|
|
'stat_name_id': stat.stat_name_id,
|
|
|
|
'start_ts': stat.start_ts,
|
|
|
|
'end_ts': stat.end_ts}
|
|
|
|
logging.debug('raw SQL query: %s', cur.query)
|
|
|
|
cur.execute(sql, params)
|
2023-03-06 18:21:56 +00:00
|
|
|
|
|
|
|
# commiting changes
|
2023-02-28 17:28:48 +00:00
|
|
|
self.__conn.commit()
|
2023-03-07 15:02:14 +00:00
|
|
|
|
|
|
|
def get_db_schema_version(self) -> Optional[int]:
|
|
|
|
sql = '''
|
|
|
|
SELECT *
|
|
|
|
FROM schema_version
|
|
|
|
LIMIT 1;
|
|
|
|
'''
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql)
|
|
|
|
val = cur.fetchone()
|
|
|
|
if not val:
|
|
|
|
return None
|
|
|
|
return int(val[0])
|
2023-03-10 18:35:12 +00:00
|
|
|
|
|
|
|
def stat_exists(self, build_task_id: int, stat_name_id: int, table_name: str) -> bool:
|
|
|
|
sql = f'''
|
|
|
|
SELECT COUNT(build_task_id)
|
|
|
|
FROM {table_name}
|
|
|
|
WHERE build_task_id = %s AND stat_name_id = %s;
|
|
|
|
'''
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql, (build_task_id, stat_name_id))
|
|
|
|
val = int(cur.fetchone()[0])
|
|
|
|
return val == 1
|
2023-03-15 10:28:12 +00:00
|
|
|
|
|
|
|
def insert_test_task(self, task: TestTaskDB):
|
|
|
|
# inserting test task itself
|
|
|
|
sql = '''
|
|
|
|
INSERT INTO test_tasks(id, build_task_id, revision, status_id, package_fullname, started_at)
|
|
|
|
VALUES (%s, %s, %s, %s, %s, %s);
|
|
|
|
'''
|
|
|
|
|
|
|
|
cur = self.__conn.cursor()
|
|
|
|
cur.execute(sql, (task.id, task.build_task_id, task.status_id,
|
|
|
|
task.package_fullname, task.started_at))
|
|
|
|
# inserting test steps stats
|
|
|
|
for ss in task.steps_stats:
|
|
|
|
sql = '''
|
|
|
|
INSERT INTO test_steps_stats (test_task_id, stat_name_id, start_ts, end_ts, success)
|
|
|
|
VALUES
|
|
|
|
(%s, %s, %s, %s, %s);
|
|
|
|
'''
|
|
|
|
cur.execute(sql, (ss.test_task_id, ss.stat_name_id,
|
|
|
|
ss.start_ts, ss.end_ts, ss.success))
|
|
|
|
|
|
|
|
# commiting changes
|
|
|
|
self.__conn.commit()
|