75 lines
2.4 KiB
Diff
75 lines
2.4 KiB
Diff
|
From 7850aea1600389beb16c7aad40bba1b76ae694c4 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Damien Ciabrini <dciabrin@redhat.com>
|
||
|
|
Date: Tue, 15 Jun 2021 20:03:20 +0200
|
||
|
|
Subject: [PATCH] podman: workaround race during container creation
|
||
|
|
|
||
|
|
podman and OCI runtime have a race that sometimes causes
|
||
|
|
a container to fail to be created and run [1] if the
|
||
|
|
cgroup to be used is not available yet. When that happens,
|
||
|
|
try to recreate it until it succeeds or the start
|
||
|
|
timeout is reached.
|
||
|
|
|
||
|
|
[1] https://bugzilla.redhat.com/show_bug.cgi?id=1972209
|
||
|
|
---
|
||
|
|
heartbeat/podman | 32 ++++++++++++++++++++++++++++++--
|
||
|
|
1 file changed, 30 insertions(+), 2 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/heartbeat/podman b/heartbeat/podman
|
||
|
|
index 5b707f3f5..034dfff76 100755
|
||
|
|
--- a/heartbeat/podman
|
||
|
|
+++ b/heartbeat/podman
|
||
|
|
@@ -358,8 +358,18 @@ run_new_container()
|
||
|
|
local rc
|
||
|
|
|
||
|
|
ocf_log info "running container $CONTAINER for the first time"
|
||
|
|
- ocf_run podman run $opts $image $cmd
|
||
|
|
+ out=$(podman run $opts $image $cmd 2>&1)
|
||
|
|
rc=$?
|
||
|
|
+
|
||
|
|
+ if [ -n "$out" ]; then
|
||
|
|
+ out="$(echo "$out" | tr -s ' \t\r\n' ' ')"
|
||
|
|
+ if [ $rc -eq 0 ]; then
|
||
|
|
+ ocf_log info "$out"
|
||
|
|
+ else
|
||
|
|
+ ocf_log err "$out"
|
||
|
|
+ fi
|
||
|
|
+ fi
|
||
|
|
+
|
||
|
|
if [ $rc -eq 125 ]; then
|
||
|
|
# If an internal podman error occurred, it might be because
|
||
|
|
# the internal storage layer still references an old container
|
||
|
|
@@ -370,6 +380,24 @@ run_new_container()
|
||
|
|
ocf_run podman rm --storage $CONTAINER
|
||
|
|
ocf_run podman run $opts $image $cmd
|
||
|
|
rc=$?
|
||
|
|
+ elif [ $rc -eq 127 ]; then
|
||
|
|
+ # rhbz#1972209: podman 3.0.x seems to be hit by a race
|
||
|
|
+ # where the cgroup is not yet set up properly when the OCI
|
||
|
|
+ # runtime configures the container. If that happens, recreate
|
||
|
|
+ # the container as long as we get the same error code or
|
||
|
|
+ # until start timeout preempts us.
|
||
|
|
+ while [ $rc -eq 127 ] && (echo "$out" | grep -q "cgroup.*scope not found") ; do
|
||
|
|
+ ocf_log warn "Internal podman error while assigning cgroup. Retrying."
|
||
|
|
+ # Arbitrary sleep to prevent consuming all CPU while looping
|
||
|
|
+ sleep 1
|
||
|
|
+ podman rm -f "$CONTAINER"
|
||
|
|
+ out=$(podman run $opts $image $cmd 2>&1)
|
||
|
|
+ rc=$?
|
||
|
|
+ done
|
||
|
|
+ # Log the created container ID if it succeeded
|
||
|
|
+ if [ $rc -eq 0 ]; then
|
||
|
|
+ ocf_log info "$out"
|
||
|
|
+ fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
return $rc
|
||
|
|
@@ -422,7 +450,7 @@ podman_start()
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ $rc -ne 0 ]; then
|
||
|
|
- ocf_exit_reason "podman failed to launch container"
|
||
|
|
+ ocf_exit_reason "podman failed to launch container (rc: $rc)"
|
||
|
|
return $OCF_ERR_GENERIC
|
||
|
|
fi
|
||
|
|
|