Compare commits
No commits in common. "fd8b88c82dd8ef5659bdce9ac7c2363a4b7b4f48" and "7543878abf6641fb49a576b34323a3f33f1094f6" have entirely different histories.
fd8b88c82d
...
7543878abf
@ -60,9 +60,7 @@ class APIclient():
|
|||||||
if task['started_at'] else None
|
if task['started_at'] else None
|
||||||
finished_at = datetime.fromisoformat(task['finished_at']+TZ_OFFSET) \
|
finished_at = datetime.fromisoformat(task['finished_at']+TZ_OFFSET) \
|
||||||
if task['finished_at'] else None
|
if task['finished_at'] else None
|
||||||
name = task['ref']['url'].split('/')[-1].replace('.git', '')
|
|
||||||
params = {'id': task['id'],
|
params = {'id': task['id'],
|
||||||
'name': name,
|
|
||||||
'build_id': build_id,
|
'build_id': build_id,
|
||||||
'started_at': started_at,
|
'started_at': started_at,
|
||||||
'finished_at': finished_at,
|
'finished_at': finished_at,
|
||||||
|
@ -35,12 +35,12 @@ class DB():
|
|||||||
|
|
||||||
def insert_buildtask(self, build_task: BuildTaskDB):
|
def insert_buildtask(self, build_task: BuildTaskDB):
|
||||||
sql = '''
|
sql = '''
|
||||||
INSERT INTO build_tasks(id, name, build_id, arch_id, started_at, finished_at, status_id)
|
INSERT INTO build_tasks(id, build_id, arch_id, started_at, finished_at, status_id)
|
||||||
VALUES (%s, %s, %s, %s, %s, %s, %s);
|
VALUES (%s, %s, %s, %s, %s, %s);
|
||||||
'''
|
'''
|
||||||
|
|
||||||
cur = self.__conn.cursor()
|
cur = self.__conn.cursor()
|
||||||
cur.execute(sql, (build_task.id, build_task.name, build_task.build_id, build_task.arch_id,
|
cur.execute(sql, (build_task.id, build_task.build_id, build_task.arch_id,
|
||||||
build_task.started_at, build_task.finished_at, build_task.status_id))
|
build_task.started_at, build_task.finished_at, build_task.status_id))
|
||||||
self.__conn.commit()
|
self.__conn.commit()
|
||||||
|
|
||||||
|
@ -86,5 +86,5 @@ def start(yml_path: str):
|
|||||||
|
|
||||||
extractor.db.close_conn()
|
extractor.db.close_conn()
|
||||||
logging.info("Extraction was finished")
|
logging.info("Extraction was finished")
|
||||||
logging.info("Sleeping for %d seconds", config.scrape_interval)
|
logging.info("Sleeping for %d seconds", config.scrape_inteval)
|
||||||
time.sleep(config.scrape_interval)
|
time.sleep(config.scrape_inteval)
|
||||||
|
@ -9,7 +9,6 @@ from .enums import ArchEnum
|
|||||||
|
|
||||||
class BuildTask(BaseModel):
|
class BuildTask(BaseModel):
|
||||||
id: int
|
id: int
|
||||||
name: str
|
|
||||||
build_id: int
|
build_id: int
|
||||||
arch: str
|
arch: str
|
||||||
started_at: Optional[datetime] = None
|
started_at: Optional[datetime] = None
|
||||||
@ -23,7 +22,6 @@ class BuildTask(BaseModel):
|
|||||||
if self.finished_at else None
|
if self.finished_at else None
|
||||||
params = {
|
params = {
|
||||||
'id': self.id,
|
'id': self.id,
|
||||||
'name': self.name,
|
|
||||||
'build_id': self.build_id,
|
'build_id': self.build_id,
|
||||||
'arch_id': ArchEnum[self.arch].value,
|
'arch_id': ArchEnum[self.arch].value,
|
||||||
'started_at': started_at,
|
'started_at': started_at,
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
from typing import Optional
|
from typing import Optional
|
||||||
from pydantic import BaseModel # pylint: disable=no-name-in-module
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|
||||||
class BuildTaskDB(BaseModel):
|
class BuildTaskDB(BaseModel):
|
||||||
@ -7,7 +7,6 @@ class BuildTaskDB(BaseModel):
|
|||||||
BuildTask as it received from/sent to database
|
BuildTask as it received from/sent to database
|
||||||
"""
|
"""
|
||||||
id: int
|
id: int
|
||||||
name: str
|
|
||||||
build_id: int
|
build_id: int
|
||||||
arch_id: int
|
arch_id: int
|
||||||
started_at: Optional[float] = None
|
started_at: Optional[float] = None
|
||||||
|
@ -27,5 +27,5 @@ class ExtractorConfig(BaseModel):
|
|||||||
api_timeout: int = Field(
|
api_timeout: int = Field(
|
||||||
description="max time in seconds to wait for API response",
|
description="max time in seconds to wait for API response",
|
||||||
default=API_DEFAULT)
|
default=API_DEFAULT)
|
||||||
scrape_interval: int = Field(description='how often (in seconds) we will extract data from ALBS',
|
scrape_inteval: int = Field(description='how often (in seconds) we will extract data from ALBS',
|
||||||
default=SCRAPE_INTERVAL_DEFAULT)
|
default=SCRAPE_INTERVAL_DEFAULT)
|
||||||
|
@ -1,58 +0,0 @@
|
|||||||
# albs_url
|
|
||||||
# url of AlmaLinux Build system
|
|
||||||
# required no
|
|
||||||
# default https://build.almalinux.org
|
|
||||||
albs_url: https://build.almalinux.org
|
|
||||||
|
|
||||||
# jwt
|
|
||||||
# JWT token to use for API calls
|
|
||||||
# Note: JWT is not required for current verison of code, just set to ""
|
|
||||||
# required: yes
|
|
||||||
jwt: ""
|
|
||||||
|
|
||||||
|
|
||||||
# db_host
|
|
||||||
# IP/hostname of database server
|
|
||||||
# required: no
|
|
||||||
# default: localhost
|
|
||||||
db_host: localhost
|
|
||||||
|
|
||||||
# db_port
|
|
||||||
# TCP port of database server
|
|
||||||
# required: no
|
|
||||||
# default: 5432
|
|
||||||
db_port: 5432
|
|
||||||
|
|
||||||
# db_username
|
|
||||||
# database user user to connect with
|
|
||||||
# required: yes
|
|
||||||
db_username: albs_analytics
|
|
||||||
|
|
||||||
|
|
||||||
# db_password
|
|
||||||
# password to connect with
|
|
||||||
# required: yes
|
|
||||||
db_password: super_secret_password
|
|
||||||
|
|
||||||
# db_name
|
|
||||||
# database name to use for storage
|
|
||||||
# required: yes
|
|
||||||
db_name: albs_analytics
|
|
||||||
|
|
||||||
|
|
||||||
# log_file
|
|
||||||
# file to write logs to
|
|
||||||
# required: no
|
|
||||||
# default: /tmp/extractor.log
|
|
||||||
log_file: /tmp/extractor.log
|
|
||||||
|
|
||||||
# data_store_days
|
|
||||||
# oldest build (in days) to keep in DB
|
|
||||||
# required: yes
|
|
||||||
data_store_days: 30
|
|
||||||
|
|
||||||
# scape_interval
|
|
||||||
# sleep time in seconds between data extraction
|
|
||||||
# required: no
|
|
||||||
# default: 3600
|
|
||||||
scrape_interval: 3600
|
|
@ -51,7 +51,6 @@ VALUES
|
|||||||
DROP TABLE IF EXISTS build_tasks CASCADE;
|
DROP TABLE IF EXISTS build_tasks CASCADE;
|
||||||
CREATE TABLE build_tasks (
|
CREATE TABLE build_tasks (
|
||||||
id INTEGER PRIMARY KEY,
|
id INTEGER PRIMARY KEY,
|
||||||
name VARCHAR(50) NOT NULL,
|
|
||||||
build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE,
|
build_id INTEGER REFERENCES builds(id) ON DELETE CASCADE,
|
||||||
arch_id INTEGER REFERENCES arch_enum(id) ON DELETE SET NULL,
|
arch_id INTEGER REFERENCES arch_enum(id) ON DELETE SET NULL,
|
||||||
status_id INTEGER REFERENCES build_task_status_enum(id) ON DELETE SET NULL,
|
status_id INTEGER REFERENCES build_task_status_enum(id) ON DELETE SET NULL,
|
||||||
|
@ -1,2 +0,0 @@
|
|||||||
0.1.0 (2023-03-01)
|
|
||||||
First version
|
|
@ -1,9 +0,0 @@
|
|||||||
certifi==2022.12.7
|
|
||||||
charset-normalizer==3.0.1
|
|
||||||
idna==3.4
|
|
||||||
psycopg2-binary==2.9.5
|
|
||||||
pydantic==1.10.5
|
|
||||||
PyYAML==6.0
|
|
||||||
requests==2.28.2
|
|
||||||
typing_extensions==4.5.0
|
|
||||||
urllib3==1.26.14
|
|
Loading…
Reference in New Issue
Block a user