1
0
mirror of https://pagure.io/fedora-qa/os-autoinst-distri-fedora.git synced 2024-12-25 19:53:08 +00:00
os-autoinst-distri-fedora/setup_repos.py
Adam Williamson d4c39fb37f setup_repos: print the reason, if download failed
It happened a few times lately and I realized it's dumb that I
can't see why.

Signed-off-by: Adam Williamson <awilliam@redhat.com>
2024-02-14 15:40:47 -08:00

219 lines
7.3 KiB
Python
Executable File

#!/usr/bin/python3
# Copyright Red Hat
#
# This file is part of os-autoinst-distri-fedora.
#
# This file is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Author: Adam Williamson <awilliam@redhat.com>
"""
Package download and repository setup script for openQA update tests.
This script uses asyncio to download the packages to be tested, and
any 'workaround' packages, concurrently, and create repositories and
repository configuration files for them. This is work that used to be
done in-line in the test scripts, but doing it that way is slow for
large multi-package updates.
"""
import argparse
import asyncio
import glob
import os
import pathlib
import shutil
import subprocess
import sys
# these are variables to make testing this script easier...change them
# to /tmp for testing
WORKAROUNDS_DIR = "/mnt/workarounds_repo"
UPDATES_DIR = "/mnt/update_repo"
UPDATES_FILE_PATH = "/mnt"
class DownloadError(Exception):
"""Exception raised when package download fails."""
pass
# thanks, https://stackoverflow.com/questions/63782892
async def run_command(*args, **kwargs):
"""
Run a command with subprocess such that we can run multiple
concurrently.
"""
# Create subprocess
process = await asyncio.create_subprocess_exec(
*args,
# stdout must a pipe to be accessible as process.stdout
stderr=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
**kwargs,
)
# Wait for the subprocess to finish
stdout, stderr = await process.communicate()
# Return retcode, stdout and stderr
return (process.returncode, stdout.decode().strip(), stderr.decode().strip())
async def download_item(item, arch, targetdir):
"""
Download something - a build or task (with koji) or an update
(with bodhi).
"""
print(f"Downloading item {item}")
if item.isdigit():
# this will be a task ID
cmd = ("koji", "download-task", f"--arch={arch}", "--arch=noarch", item)
elif item.startswith("FEDORA-"):
# this is a Bodhi update ID
cmd = ("bodhi", "updates", "download", "--arch", arch, "--updateid", item)
else:
# assume it's an NVR
cmd = ("koji", "download-build", f"--arch={arch}", "--arch=noarch", item)
# do the download and check for failure
(retcode, _, stderr) = await run_command(*cmd, cwd=targetdir)
if retcode:
# "No .*available for {nvr}" indicates there are no
# packages for this arch in the build
if not f"available for {item}" in stderr:
print(f"Downloading {item} failed: {stderr}")
return item
return False
async def create_workarounds_repo(workarounds, arch, config):
"""Set up the workarounds repository."""
shutil.rmtree(WORKAROUNDS_DIR, ignore_errors=True)
os.makedirs(WORKAROUNDS_DIR)
rets = []
if workarounds:
for i in range(0, len(workarounds), 20):
tasks = [
asyncio.create_task(download_item(item, arch, WORKAROUNDS_DIR))
for item in workarounds[i : i + 20]
]
rets.extend(await asyncio.gather(*tasks))
subprocess.run(["createrepo", "."], cwd=WORKAROUNDS_DIR, check=True)
if config:
with open("/etc/yum.repos.d/workarounds.repo", "w", encoding="utf-8") as repofh:
repofh.write(
"[workarounds]\nname=Workarounds repo\n"
"baseurl=file:///mnt/workarounds_repo\n"
"enabled=1\nmetadata_expire=1\ngpgcheck=0"
)
return [ret for ret in rets if ret]
async def create_updates_repo(items, arch, config):
"""Set up the updates/task repository."""
# we do not recreate the directory as the test code has to do that
# since it has to mount it, before we run
rets = []
for i in range(0, len(items), 20):
tasks = [
asyncio.create_task(download_item(item, arch, UPDATES_DIR))
for item in items[i : i + 20]
]
rets.extend(await asyncio.gather(*tasks))
subprocess.run(["createrepo", "."], cwd=UPDATES_DIR, check=True)
if not glob.glob(f"{UPDATES_DIR}/*.rpm"):
pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgnames.txt").touch()
pathlib.Path(f"{UPDATES_FILE_PATH}/updatepkgs.txt").touch()
else:
cmd = "rpm -qp *.rpm --qf '%{SOURCERPM} %{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE}\n' | "
cmd += f"sort -u > {UPDATES_FILE_PATH}/updatepkgs.txt"
subprocess.run(cmd, shell=True, check=True, cwd=UPDATES_DIR)
# also log just the binary package names: this is so we can check
# later whether any package from the update *should* have been
# installed, but was not
subprocess.run(
"rpm -qp *.rpm --qf '%{NAME} ' > "
+ f"{UPDATES_FILE_PATH}/updatepkgnames.txt",
shell=True,
check=True,
cwd=UPDATES_DIR,
)
if config:
with open("/etc/yum.repos.d/advisory.repo", "w", encoding="utf-8") as repofh:
repofh.write(
"[advisory]\nname=Advisory repo\nbaseurl=file:///mnt/update_repo\n"
"enabled=1\nmetadata_expire=3600\ngpgcheck=0"
)
return [ret for ret in rets if ret]
def commalist(string):
"""Separate a string on commas."""
return string.split(",")
def parse_args():
"""Parse CLI args with argparse."""
parser = argparse.ArgumentParser(
description="Packager downloader script for openQA tests"
)
parser.add_argument("arch", help="Architecture")
parser.add_argument(
"--workarounds",
"-w",
type=commalist,
help="Comma-separated list of workaround packages",
)
parser.add_argument(
"--updates",
"-u",
type=commalist,
help="Comma-separated list of update/task packages",
)
parser.add_argument(
"--configs", "-c", action="store_true", help="Write repo config files"
)
args = parser.parse_args()
if not (args.workarounds or args.updates):
parser.error("At least one of workarounds or updates package lists is required")
return args
async def main():
"""Do the thing!"""
args = parse_args()
tasks = []
if args.workarounds:
tasks.append(
asyncio.create_task(
create_workarounds_repo(args.workarounds, args.arch, args.configs)
)
)
if args.updates:
tasks.append(
asyncio.create_task(
create_updates_repo(args.updates, args.arch, args.configs)
)
)
failed = []
rets = await asyncio.gather(*tasks, return_exceptions=True)
for ret in rets:
if isinstance(ret, Exception):
raise ret
failed.extend(ret)
if failed:
sys.exit(f"Download of item(s) {', '.join(failed)} failed!")
asyncio.run(main())