From 936d193413375153a671f709371ed8bdd58730de Mon Sep 17 00:00:00 2001
From: "Brian C. Lane" <bcl@redhat.com>
Date: Fri, 8 Jun 2018 17:05:38 -0700
Subject: [PATCH] Retry losetup if loop_attach fails

It appears that sometimes the loop device doesn't get setup properly,
this may be a race with other users of loop devices on the system, or
some other mechanism that isn't understood.

To try and prevent total failure when this happens this patch retries
the loop setup 3 times before giving up. Previously it would wait for
the loop device to appear (checking 5 times), that operation is now
executed 3 times with a new losetup attempt each time.

Resolves: rhbz#1589084
(cherry picked from commit c746e8b0c376006b80fa08aadeeaed71530f7157)
---
 src/pylorax/imgutils.py | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py
index a5066e3b..5c5e3cf5 100644
--- a/src/pylorax/imgutils.py
+++ b/src/pylorax/imgutils.py
@@ -179,13 +179,34 @@ def loop_waitfor(loop_dev, outfile):
     raise RuntimeError("Unable to setup %s on %s" % (loop_dev, outfile))
 
 def loop_attach(outfile):
-    '''Attach a loop device to the given file. Return the loop device name.
-    Raises CalledProcessError if losetup fails.'''
-    dev = runcmd_output(["losetup", "--find", "--show", outfile])
+    """Attach a loop device to the given file. Return the loop device name.
 
-    # Sometimes the loop device isn't ready yet, make extra sure before returning
-    loop_waitfor(dev.strip(), outfile)
-    return dev.strip()
+    On rare occasions it appears that the device never shows up, some experiments
+    seem to indicate that it may be a race with another process using /dev/loop* devices.
+
+    So we now try 3 times before actually failing.
+
+    Raises CalledProcessError if losetup fails.
+    """
+    retries = 0
+    while True:
+        try:
+            retries += 1
+            dev = runcmd_output(["losetup", "--find", "--show", outfile]).strip()
+
+            # Sometimes the loop device isn't ready yet, make extra sure before returning
+            loop_waitfor(dev, outfile)
+        except CalledProcessError:
+            # Problems running losetup are always errors, raise immediately
+            raise
+        except RuntimeError as e:
+            # Try to setup the loop device 3 times
+            if retries == 3:
+                logger.error("loop_attach failed, retries exhausted.")
+                raise
+            logger.debug("Try %d failed, %s did not appear.", retries, dev)
+        break
+    return dev
 
 def loop_detach(loopdev):
     '''Detach the given loop device. Return False on failure.'''