mirror of
https://pagure.io/fedora-qa/os-autoinst-distri-fedora.git
synced 2024-09-08 17:47:22 +00:00
It happened a few times lately and I realized it's dumb that I can't see why. Signed-off-by: Adam Williamson <awilliam@redhat.com>
219 lines
7.3 KiB
Python
Executable File
219 lines
7.3 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
# Copyright Red Hat
|
|
#
|
|
# This file is part of os-autoinst-distri-fedora.
|
|
#
|
|
# This file is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# Author: Adam Williamson <awilliam@redhat.com>
|
|
|
|
"""
|
|
Package download and repository setup script for openQA update tests.
|
|
This script uses asyncio to download the packages to be tested, and
|
|
any 'workaround' packages, concurrently, and create repositories and
|
|
repository configuration files for them. This is work that used to be
|
|
done in-line in the test scripts, but doing it that way is slow for
|
|
large multi-package updates.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import glob
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
# these are variables to make testing this script easier...change them
|
|
# to /tmp for testing
|
|
WORKAROUNDS_DIR = "/mnt/workarounds_repo"
|
|
UPDATES_DIR = "/mnt/update_repo"
|
|
UPDATES_FILE_PATH = "/mnt"
|
|
|
|
|
|
class DownloadError(Exception):
|
|
"""Exception raised when package download fails."""
|
|
pass
|
|
|
|
|
|
# thanks, https://stackoverflow.com/questions/63782892
|
|
async def run_command(*args, **kwargs):
|
|
"""
|
|
Run a command with subprocess such that we can run multiple
|
|
concurrently.
|
|
"""
|
|
# Create subprocess
|
|
process = await asyncio.create_subprocess_exec(
|
|
*args,
|
|
# stdout must a pipe to be accessible as process.stdout
|
|
stderr=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
**kwargs,
|
|
)
|
|
# Wait for the subprocess to finish
|
|
stdout, stderr = await process.communicate()
|
|
# Return retcode, stdout and stderr
|
|
return (process.returncode, stdout.decode().strip(), stderr.decode().strip())
|
|
|
|
|
|
async def download_item(item, arch, targetdir):
|
|
"""
|
|
Download something - a build or task (with koji) or an update
|
|
(with bodhi).
|
|
"""
|
|
print(f"Downloading item {item}")
|
|
if item.isdigit():
|
|
# this will be a task ID
|
|
cmd = ("koji", "download-task", f"--arch={arch}", "--arch=noarch", item)
|
|
elif item.startswith("FEDORA-"):
|
|
# this is a Bodhi update ID
|
|
cmd = ("bodhi", "updates", "download", "--arch", arch, "--updateid", item)
|
|
else:
|
|
# assume it's an NVR
|
|
cmd = ("koji", "download-build", f"--arch={arch}", "--arch=noarch", item)
|
|
# do the download and check for failure
|
|
(retcode, _, stderr) = await run_command(*cmd, cwd=targetdir)
|
|
if retcode:
|
|
# "No .*available for {nvr}" indicates there are no
|
|
# packages for this arch in the build
|
|
if not f"available for {item}" in stderr:
|
|
print(f"Downloading {item} failed: {stderr}")
|
|
return item
|
|
return False
|
|
|
|
|
|
async def create_workarounds_repo(workarounds, arch, config):
|
|
"""Set up the workarounds repository."""
|
|
shutil.rmtree(WORKAROUNDS_DIR, ignore_errors=True)
|
|
os.makedirs(WORKAROUNDS_DIR)
|
|
rets = []
|
|
if workarounds:
|
|
for i in range(0, len(workarounds), 20):
|
|
tasks = [
|
|
asyncio.create_task(download_item(item, arch, WORKAROUNDS_DIR))
|
|
for item in workarounds[i : i + 20]
|
|
]
|
|
rets.extend(await asyncio.gather(*tasks))
|
|
subprocess.run(["createrepo", "."], cwd=WORKAROUNDS_DIR, check=True)
|
|
if config:
|
|
with open("/etc/yum.repos.d/workarounds.repo", "w", encoding="utf-8") as repofh:
|
|
repofh.write(
|
|
"[workarounds]\nname=Workarounds repo\n"
|
|
"baseurl=file:///mnt/workarounds_repo\n"
|
|
"enabled=1\nmetadata_expire=1\ngpgcheck=0"
|
|
)
|
|
return [ret for ret in rets if ret]
|
|
|
|
|
|
async def create_updates_repo(items, arch, config):
|
|
"""Set up the updates/task repository."""
|
|
# we do not recreate the directory as the test code has to do that
|
|
# since it has to mount it, before we run
|
|
rets = []
|
|
for i in range(0, len(items), 20):
|
|
tasks = [
|
|
asyncio.create_task(download_item(item, arch, UPDATES_DIR))
|
|
for item in items[i : i + 20]
|
|
]
|
|
rets.extend(await asyncio.gather(*tasks))
|
|
subprocess.run(["createrepo", "."], cwd=UPDATES_DIR, check=True)
|
|
if not glob.glob(f"{UPDATES_DIR}/*.rpm"):
|
|
pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgnames.txt").touch()
|
|
pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgs.txt").touch()
|
|
else:
|
|
cmd = "rpm -qp *.rpm --qf '%{SOURCERPM} %{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE}\n' | "
|
|
cmd += f"sort -u > {UPDATES_FILE_PATH}/updatepkgs.txt"
|
|
subprocess.run(cmd, shell=True, check=True, cwd=UPDATES_DIR)
|
|
# also log just the binary package names: this is so we can check
|
|
# later whether any package from the update *should* have been
|
|
# installed, but was not
|
|
subprocess.run(
|
|
"rpm -qp *.rpm --qf '%{NAME} ' > "
|
|
+ f"{UPDATES_FILE_PATH}/updatepkgnames.txt",
|
|
shell=True,
|
|
check=True,
|
|
cwd=UPDATES_DIR,
|
|
)
|
|
if config:
|
|
with open("/etc/yum.repos.d/advisory.repo", "w", encoding="utf-8") as repofh:
|
|
repofh.write(
|
|
"[advisory]\nname=Advisory repo\nbaseurl=file:///mnt/update_repo\n"
|
|
"enabled=1\nmetadata_expire=3600\ngpgcheck=0"
|
|
)
|
|
return [ret for ret in rets if ret]
|
|
|
|
|
|
def commalist(string):
|
|
"""Separate a string on commas."""
|
|
return string.split(",")
|
|
|
|
|
|
def parse_args():
|
|
"""Parse CLI args with argparse."""
|
|
parser = argparse.ArgumentParser(
|
|
description="Packager downloader script for openQA tests"
|
|
)
|
|
parser.add_argument("arch", help="Architecture")
|
|
parser.add_argument(
|
|
"--workarounds",
|
|
"-w",
|
|
type=commalist,
|
|
help="Comma-separated list of workaround packages",
|
|
)
|
|
parser.add_argument(
|
|
"--updates",
|
|
"-u",
|
|
type=commalist,
|
|
help="Comma-separated list of update/task packages",
|
|
)
|
|
parser.add_argument(
|
|
"--configs", "-c", action="store_true", help="Write repo config files"
|
|
)
|
|
args = parser.parse_args()
|
|
if not (args.workarounds or args.updates):
|
|
parser.error("At least one of workarounds or updates package lists is required")
|
|
return args
|
|
|
|
|
|
async def main():
|
|
"""Do the thing!"""
|
|
args = parse_args()
|
|
|
|
tasks = []
|
|
if args.workarounds:
|
|
tasks.append(
|
|
asyncio.create_task(
|
|
create_workarounds_repo(args.workarounds, args.arch, args.configs)
|
|
)
|
|
)
|
|
if args.updates:
|
|
tasks.append(
|
|
asyncio.create_task(
|
|
create_updates_repo(args.updates, args.arch, args.configs)
|
|
)
|
|
)
|
|
failed = []
|
|
rets = await asyncio.gather(*tasks, return_exceptions=True)
|
|
for ret in rets:
|
|
if isinstance(ret, Exception):
|
|
raise ret
|
|
failed.extend(ret)
|
|
if failed:
|
|
sys.exit(f"Download of item(s) {', '.join(failed)} failed!")
|
|
|
|
|
|
asyncio.run(main())
|