From ec62bb2a447a205bf5a77a4c6d414e056a4bf79c Mon Sep 17 00:00:00 2001 From: Josef Skladanka Date: Wed, 28 Jan 2015 11:00:58 +0100 Subject: [PATCH] Automated reporting using relval --- createhdds.sh | 0 openqa_trigger/conf_test_suites.py | 111 ++++++++++++++++++ .../openqa_trigger.py | 0 openqa_trigger/report_job_results.py | 84 +++++++++++++ 4 files changed, 195 insertions(+) mode change 100644 => 100755 createhdds.sh create mode 100644 openqa_trigger/conf_test_suites.py rename openqa_trigger.py => openqa_trigger/openqa_trigger.py (100%) mode change 100644 => 100755 create mode 100644 openqa_trigger/report_job_results.py diff --git a/createhdds.sh b/createhdds.sh old mode 100644 new mode 100755 diff --git a/openqa_trigger/conf_test_suites.py b/openqa_trigger/conf_test_suites.py new file mode 100644 index 0000000..50f61dc --- /dev/null +++ b/openqa_trigger/conf_test_suites.py @@ -0,0 +1,111 @@ +TESTCASES = { + "QA:Testcase_Boot_default_install Server offline": { + "section": 'Default boot', + "env": "$RUNARCH$", + "type": "Installation", + }, + "QA:Testcase_install_to_VirtIO": { + "section": "Storage devices", + "env": "x86", + "type": "Installation", + }, + "QA:Testcase_partitioning_guided_empty": { + "section": "Guided storage configuration", + "env": "x86 BIOS", + "type": "Installation", + }, + "QA:Testcase_Anaconda_User_Interface_Graphical": { + "section": "User interface", + "env": "Result", + "type": "Installation", + }, + "QA:Testcase_Anaconda_user_creation": { + "section": "Miscellaneous", + "env": "x86", + "type": "Installation", + }, + "QA:Testcase_install_to_PATA": { + "section": "Storage devices", + "env": "x86", + "type": "Installation", + }, + "QA:Testcase_partitioning_guided_delete_all": { + "section": "Guided storage configuration", + "env": "x86 BIOS", + "type": "Installation", + }, + "QA:Testcase_install_to_SATA": { + "section": "Storage devices", + "env": "x86", + "type": "Installation", + }, + "QA:Testcase_partitioning_guided_multi_select": { + "section": "Guided storage configuration", + "env": "x86 BIOS", + "type": "Installation", + }, + "QA:Testcase_install_to_SCSI": { + "section": "Storage devices", + "env": "x86", + "type": "Installation", + }, + "QA:Testcase_Anaconda_updates.img_via_URL": { + "section": "Miscellaneous ", + "env": "Result", + "type": "Installation", + }, + "QA:Testcase_kickstart_user_creation": { + "section": "Kickstart", + "env": "Result", + "type": "Installation", + }, + "QA:Testcase_Kickstart_Http_Server_Ks_Cfg": { + "section": "Kickstart", + "env": "Result", + "type": "Installation", + }, +# "": { +# "section": "", +# "env": "x86", +# "type": "Installation", +# }, + } + + +TESTSUITES = { + "server_simple":[ + "QA:Testcase_Boot_default_install Server offline", + "QA:Testcase_install_to_VirtIO", + "QA:Testcase_Anaconda_User_Interface_Graphical", + "QA:Testcase_Anaconda_user_creation", + ], + "server_delete_pata":[ + "QA:Testcase_Boot_default_install Server offline", + "QA:Testcase_install_to_PATA", + "QA:Testcase_partitioning_guided_delete_all", + "QA:Testcase_Anaconda_User_Interface_Graphical", + "QA:Testcase_Anaconda_user_creation", + ], + "server_sata_multi":[ + "QA:Testcase_Boot_default_install Server offline", + "QA:Testcase_install_to_SATA", + "QA:Testcase_partitioning_guided_multi_select", + "QA:Testcase_Anaconda_User_Interface_Graphical", + "QA:Testcase_Anaconda_user_creation", + ], + "server_scsi_updatesimg":[ + "QA:Testcase_Boot_default_install Server offline", + "QA:Testcase_install_to_SCSI", + "QA:Testcase_partitioning_guided_empty", + "QA:Testcase_Anaconda_updates.img_via_URL", + "QA:Testcase_Anaconda_User_Interface_Graphical", + "QA:Testcase_Anaconda_user_creation", + ], + "server_kickstart_user_creation":[ + "QA:Testcase_install_to_VirtIO", + "QA:Testcase_Anaconda_user_creation", + "QA:Testcase_kickstart_user_creation", + "QA:Testcase_Kickstart_Http_Server_Ks_Cfg", + ], + } + diff --git a/openqa_trigger.py b/openqa_trigger/openqa_trigger.py old mode 100644 new mode 100755 similarity index 100% rename from openqa_trigger.py rename to openqa_trigger/openqa_trigger.py diff --git a/openqa_trigger/report_job_results.py b/openqa_trigger/report_job_results.py new file mode 100644 index 0000000..91974ec --- /dev/null +++ b/openqa_trigger/report_job_results.py @@ -0,0 +1,84 @@ +import requests +import argparse +import os +import time +import conf_test_suites + + +API_ROOT = "http://10.34.28.126/api/v1" +SLEEPTIME = 60 + + +def get_passed_testcases(job_ids): + """ + job_ids ~ list of int (job ids) + Returns ~ list of str - names of passed testcases + """ + running_jobs = dict([(job_id, "%s/jobs/%s" % (API_ROOT, job_id)) for job_id in job_ids]) + finished_jobs = {} + + while running_jobs: + for job_id, url in running_jobs.items(): + job_state = requests.get(url).json()['job'] + if job_state['state'] == 'done': + finished_jobs[job_id] = job_state + del running_jobs[job_id] + if running_jobs: + time.sleep(SLEEPTIME) + + passed_testcases = {} # key = VERSION_BUILD_ARCH + for job_id in job_ids: + job = finished_jobs[job_id] + if job['result'] =='passed': + key = (job['settings']['VERSION'], job['settings']['FLAVOR'], job['settings'].get('BUILD', None), job['settings']['ARCH']) + passed_testcases.setdefault(key, []) + passed_testcases[key].extend(conf_test_suites.TESTSUITES[job['settings']['TEST']]) + + for key, value in passed_testcases.iteritems(): + passed_testcases[key] = sorted(list(set(value))) + return passed_testcases + + +def get_relval_commands(passed_testcases): + relval_template = "relval report-results --unmanned --result pass" + commands = [] + for key in passed_testcases: + cmd_ = relval_template + version, _, build, arch = key + + # FIXME remove after debugging + build = "22 20150124" + + if version == 'rawhide': + cmd_ += ' --release "%s" --date "%s"' % tuple(build.split()) #"22 20150110" + elif version == 'branched': + cmd_ += ' --release "%s" --milestone "%s" --compose "%s"' % tuple(build.split()) #"22 Alpha TC1" + + for tc_name in passed_testcases[key]: + testcase = conf_test_suites.TESTCASES[tc_name] + tc_env = arch if testcase['env'] == '$RUNARCH$' else testcase['env'] + tc_type = testcase['type'] + tc_section = testcase['section'] + + commands.append('%s --env "%s" --testtype "%s" --section "%s" --testcase "%s"' % (cmd_, tc_env, tc_type, tc_section, tc_name)) + return commands + + +def report_results(job_ids): + commands = get_relval_commands(get_passed_testcases(job_ids)) + for command in commands: + os.system(command) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Evaluate per-testcase results from OpenQA job runs") + parser.add_argument('jobs', type=int, nargs='+') + + args = parser.parse_args() + + passed_testcases = get_passed_testcases(args.jobs) + + import pprint + pprint.pprint(passed_testcases) + pprint.pprint(get_relval_commands(passed_testcases)) +