From 660c8bc2b3e3f18d9d0a83820e69a30ef769d20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lubom=C3=ADr=20Sedl=C3=A1=C5=99?= Date: Fri, 20 Nov 2015 10:53:55 +0100 Subject: [PATCH] Remove computing checksums from phases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These phases were computing checksums for images: * buildinstall * createiso * image_build * live_images * product_img In each phase the checksummed thing would ultimately end-up in image manifest. Signed-off-by: Lubomír Sedlář --- pungi/phases/buildinstall.py | 14 +----------- pungi/phases/createiso.py | 14 ++---------- pungi/phases/image_build.py | 41 +----------------------------------- pungi/phases/live_images.py | 6 +----- pungi/phases/product_img.py | 3 --- pungi/wrappers/iso.py | 8 ------- 6 files changed, 5 insertions(+), 81 deletions(-) diff --git a/pungi/phases/buildinstall.py b/pungi/phases/buildinstall.py index 08ce03d0..1a297755 100644 --- a/pungi/phases/buildinstall.py +++ b/pungi/phases/buildinstall.py @@ -24,7 +24,7 @@ import shutil import re from kobo.threads import ThreadPool, WorkerThread -from kobo.shortcuts import run, read_checksum_file, relative_path +from kobo.shortcuts import run, relative_path from productmd.images import Image from pungi.arch import get_valid_arches @@ -266,11 +266,8 @@ def symlink_boot_iso(compose, arch, variant): iso = IsoWrapper() implant_md5 = iso.get_implanted_md5(new_boot_iso_path) - # compute md5sum, sha1sum, sha256sum iso_name = os.path.basename(new_boot_iso_path) iso_dir = os.path.dirname(new_boot_iso_path) - for cmd in iso.get_checksum_cmds(iso_name): - run(cmd, workdir=iso_dir) # create iso manifest run(iso.get_manifest_cmd(iso_name), workdir=iso_dir) @@ -285,15 +282,6 @@ def symlink_boot_iso(compose, arch, variant): img.format = "iso" img.disc_number = 1 img.disc_count = 1 - for checksum_type in ("md5", "sha1", "sha256"): - checksum_path = new_boot_iso_path + ".%sSUM" % checksum_type.upper() - checksum_value = None - if os.path.isfile(checksum_path): - checksum_value, iso_name = read_checksum_file(checksum_path)[0] - if iso_name != os.path.basename(img.path): - # a bit paranoind check - this should never happen - raise ValueError("Image name doesn't match checksum: %s" % checksum_path) - img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) img.bootable = True img.implant_md5 = implant_md5 try: diff --git a/pungi/phases/createiso.py b/pungi/phases/createiso.py index ad78234b..40bca6d5 100644 --- a/pungi/phases/createiso.py +++ b/pungi/phases/createiso.py @@ -21,11 +21,10 @@ import pipes import random import shutil -import koji import productmd.treeinfo from productmd.images import Image from kobo.threads import ThreadPool, WorkerThread -from kobo.shortcuts import run, read_checksum_file, relative_path +from kobo.shortcuts import run, relative_path from pungi.wrappers.iso import IsoWrapper from pungi.wrappers.createrepo import CreaterepoWrapper @@ -198,7 +197,7 @@ class CreateIsoThread(WorkerThread): try: # remove incomplete ISO os.unlink(cmd["iso_path"]) - # TODO: remove jigdo & template & checksums + # TODO: remove jigdo & template except OSError: pass @@ -274,15 +273,6 @@ class CreateIsoThread(WorkerThread): img.format = "iso" img.disc_number = cmd["disc_num"] img.disc_count = cmd["disc_count"] - for checksum_type in ("md5", "sha1", "sha256"): - checksum_path = cmd["iso_path"] + ".%sSUM" % checksum_type.upper() - checksum_value = None - if os.path.isfile(checksum_path): - checksum_value, iso_name = read_checksum_file(checksum_path)[0] - if iso_name != os.path.basename(img.path): - # a bit paranoind check - this should never happen - raise ValueError("Image name doesn't match checksum: %s" % checksum_path) - img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) img.bootable = cmd["bootable"] img.implant_md5 = iso.get_implanted_md5(cmd["iso_path"]) try: diff --git a/pungi/phases/image_build.py b/pungi/phases/image_build.py index 95bbd5aa..0234bb7e 100644 --- a/pungi/phases/image_build.py +++ b/pungi/phases/image_build.py @@ -3,18 +3,16 @@ import os import time -import pipes from pungi.util import get_arch_variant_data from pungi.phases.base import PhaseBase from pungi.linker import Linker from pungi.paths import translate_path from pungi.wrappers.kojiwrapper import KojiWrapper -from pungi.wrappers.iso import IsoWrapper -from kobo.shortcuts import run, read_checksum_file from kobo.threads import ThreadPool, WorkerThread from productmd.images import Image + class ImageBuildPhase(PhaseBase): """class for wrapping up koji image-build""" name = "image_build" @@ -69,7 +67,6 @@ class CreateImageBuildThread(WorkerThread): mounts = [compose.topdir] if "mount" in cmd: mounts.append(cmd["mount"]) - runroot = compose.conf.get("runroot", False) log_file = compose.paths.log.log_file(cmd["image_conf"]["arches"], "imagebuild-%s-%s-%s" % (cmd["image_conf"]["arches"], cmd["image_conf"]["variant"], cmd['image_conf']['format'].replace(",","-"))) msg = "Creating %s image (arch: %s, variant: %s)" % (cmd["image_conf"]["format"].replace(",","-"), cmd["image_conf"]["arches"], cmd["image_conf"]["variant"]) self.pool.log_info("[BEGIN] %s" % msg) @@ -113,34 +110,6 @@ class CreateImageBuildThread(WorkerThread): image_dest = os.path.join(cmd["image_dir"], os.path.basename(image_info['filename'])) linker.link(image_info['filename'], image_dest, link_type=cmd["link_type"]) - iso = IsoWrapper(logger=compose._logger) # required for checksums only - checksum_cmd = ["cd %s" % pipes.quote(os.path.dirname(image_dest))] - checksum_cmd.extend(iso.get_checksum_cmds(os.path.basename(image_dest))) - checksum_cmd = " && ".join(checksum_cmd) - - if runroot: - packages = ["coreutils", "genisoimage", "isomd5sum", "jigdo", "strace", "lsof"] - runroot_channel = compose.conf.get("runroot_channel", None) - runroot_tag = compose.conf["runroot_tag"] - koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, cmd["image_conf"]["arches"], checksum_cmd, channel=runroot_channel, use_shell=True, task_id=True, packages=packages, mounts=mounts) - - # avoid race conditions? - # Kerberos authentication failed: Permission denied in replay cache code (-1765328215) - time.sleep(num * 3) - - output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file) - if output["retcode"] != 0: - self.fail(compose, cmd) - raise RuntimeError("Runroot task failed: %s. See %s for more details." % (output["task_id"], log_file)) - - else: - # run locally - try: - run(checksum_cmd, show_cmd=True, logfile=log_file) - except: - self.fail(compose, cmd) - raise - # Update image manifest img = Image(compose.im) img.type = image_info['type'] @@ -151,14 +120,6 @@ class CreateImageBuildThread(WorkerThread): img.arch = cmd["image_conf"]["arches"] # arches should be always single arch img.disc_number = 1 # We don't expect multiple disks img.disc_count = 1 - for checksum_type in ("md5", "sha1", "sha256"): - checksum_path = image_dest + ".%sSUM" % checksum_type.upper() - checksum_value = None - if os.path.isfile(checksum_path): - checksum_value, image_name = read_checksum_file(checksum_path)[0] - if image_name != os.path.basename(img.path): - raise ValueError("Image name doesn't match checksum: %s" % checksum_path) - img.add_checksum(compose.paths.compose.topdir(), checksum_type=checksum_type, checksum_value=checksum_value) img.bootable = False # named keywords due portability (old productmd used arch, variant ... while new one uses variant, arch compose.im.add(variant=cmd["image_conf"]["variant"].uid, arch=cmd["image_conf"]["arches"], image=img) diff --git a/pungi/phases/live_images.py b/pungi/phases/live_images.py index 9e209b2b..aacc1290 100644 --- a/pungi/phases/live_images.py +++ b/pungi/phases/live_images.py @@ -136,9 +136,6 @@ class LiveImagesPhase(PhaseBase): chdir_cmd = "cd %s" % pipes.quote(iso_dir) cmd["cmd"].append(chdir_cmd) - # compute md5sum, sha1sum, sha256sum - cmd["cmd"].extend(iso.get_checksum_cmds(iso_name)) - # create iso manifest cmd["cmd"].append(iso.get_manifest_cmd(iso_name)) @@ -163,7 +160,6 @@ class CreateLiveImageThread(WorkerThread): try: # remove (possibly?) incomplete ISO os.unlink(cmd["iso_path"]) - # TODO: remove checksums except OSError: pass @@ -207,7 +203,7 @@ class CreateLiveImageThread(WorkerThread): for rpm_path in rpm_paths: shutil.copy2(rpm_path, cmd["wrapped_rpms_path"]) - # write checksum and manifest + # write manifest run(cmd["cmd"]) self.pool.log_info("[DONE ] %s" % msg) diff --git a/pungi/phases/product_img.py b/pungi/phases/product_img.py index fd08981f..a39ac926 100644 --- a/pungi/phases/product_img.py +++ b/pungi/phases/product_img.py @@ -262,9 +262,6 @@ def rebuild_boot_iso(compose, arch, variant, package_sets): shutil.rmtree(tmp_dir) shutil.rmtree(mount_dir) - # .treeinfo is written after productimg phase - # -> checksums should match - # -> no need to write/modify it here compose.log_info("[DONE ] %s" % msg) diff --git a/pungi/wrappers/iso.py b/pungi/wrappers/iso.py index 8b3ebdf4..2b1fc247 100644 --- a/pungi/wrappers/iso.py +++ b/pungi/wrappers/iso.py @@ -206,14 +206,6 @@ class IsoWrapper(kobo.log.LoggingBase): result = line.rsplit(":")[-1].strip() return result - def get_checksum_cmds(self, iso_name, checksum_types=None): - checksum_types = checksum_types or ["md5", "sha1", "sha256"] - result = [] - for checksum_type in checksum_types: - cmd = "%ssum -b %s > %s.%sSUM" % (checksum_type.lower(), pipes.quote(iso_name), pipes.quote(iso_name), checksum_type.upper()) - result.append(cmd) - return result - def get_manifest_cmd(self, iso_name): return "isoinfo -R -f -i %s | grep -v '/TRANS.TBL$' | sort >> %s.manifest" % (pipes.quote(iso_name), pipes.quote(iso_name))