91 lines
3.8 KiB
Diff
91 lines
3.8 KiB
Diff
|
From f11bbe7f04a48eebcb446e283820d7592f76cf86 Mon Sep 17 00:00:00 2001
|
||
|
|
From: Johnson Shi <Johnson.Shi@microsoft.com>
|
||
|
|
Date: Thu, 25 Mar 2021 07:20:10 -0700
|
||
|
|
Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
|
||
|
|
retries (#842)
|
||
|
|
|
||
|
|
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||
|
|
RH-MergeRequest: 45: Add support for userdata on Azure from IMDS
|
||
|
|
RH-Commit: [2/7] e8f8bb658b629a8444bd2ba19f109952acf33311
|
||
|
|
RH-Bugzilla: 2023940
|
||
|
|
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||
|
|
RH-Acked-by: Mohamed Gamal Morsy <mmorsy@redhat.com>
|
||
|
|
|
||
|
|
Ensure that the Azure helper's http handler sleeps a fixed duration
|
||
|
|
between retry failure attempts. The http handler will sleep a fixed
|
||
|
|
duration between failed attempts regardless of whether the attempt
|
||
|
|
failed due to (1) request timing out or (2) instant failure (no
|
||
|
|
timeout).
|
||
|
|
|
||
|
|
Due to certain platform issues, the http request to the Azure endpoint
|
||
|
|
may instantly fail without reaching the http timeout duration. Without
|
||
|
|
sleeping a fixed duration in between retry attempts, the http handler
|
||
|
|
will loop through the max retry attempts quickly. This causes the
|
||
|
|
communication between cloud-init and the Azure platform to be less
|
||
|
|
resilient due to the short total duration if there is no sleep in
|
||
|
|
between retries.
|
||
|
|
---
|
||
|
|
cloudinit/sources/helpers/azure.py | 2 ++
|
||
|
|
tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
|
||
|
|
2 files changed, 11 insertions(+), 2 deletions(-)
|
||
|
|
|
||
|
|
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
|
||
|
|
index d3055d08..03e7156b 100755
|
||
|
|
--- a/cloudinit/sources/helpers/azure.py
|
||
|
|
+++ b/cloudinit/sources/helpers/azure.py
|
||
|
|
@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
|
||
|
|
|
||
|
|
max_readurl_attempts = 240
|
||
|
|
default_readurl_timeout = 5
|
||
|
|
+ sleep_duration_between_retries = 5
|
||
|
|
periodic_logging_attempts = 12
|
||
|
|
|
||
|
|
if 'timeout' not in kwargs:
|
||
|
|
@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
|
||
|
|
'attempt %d with exception: %s' %
|
||
|
|
(url, attempt, e),
|
||
|
|
logger_func=LOG.debug)
|
||
|
|
+ time.sleep(sleep_duration_between_retries)
|
||
|
|
|
||
|
|
raise exc
|
||
|
|
|
||
|
|
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
|
||
|
|
index b8899807..63482c6c 100644
|
||
|
|
--- a/tests/unittests/test_datasource/test_azure_helper.py
|
||
|
|
+++ b/tests/unittests/test_datasource/test_azure_helper.py
|
||
|
|
@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||
|
|
|
||
|
|
max_readurl_attempts = 240
|
||
|
|
default_readurl_timeout = 5
|
||
|
|
+ sleep_duration_between_retries = 5
|
||
|
|
periodic_logging_attempts = 12
|
||
|
|
|
||
|
|
def setUp(self):
|
||
|
|
@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||
|
|
self.m_readurl = patches.enter_context(
|
||
|
|
mock.patch.object(
|
||
|
|
azure_helper.url_helper, 'readurl', mock.MagicMock()))
|
||
|
|
- patches.enter_context(
|
||
|
|
- mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
|
||
|
|
+ self.m_sleep = patches.enter_context(
|
||
|
|
+ mock.patch.object(azure_helper.time, 'sleep', autospec=True))
|
||
|
|
|
||
|
|
def test_http_with_retries(self):
|
||
|
|
self.m_readurl.return_value = 'TestResp'
|
||
|
|
@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||
|
|
self.m_readurl.call_count,
|
||
|
|
self.periodic_logging_attempts + 1)
|
||
|
|
|
||
|
|
+ # Ensure that cloud-init did sleep between each failed request
|
||
|
|
+ self.assertEqual(
|
||
|
|
+ self.m_sleep.call_count,
|
||
|
|
+ self.periodic_logging_attempts)
|
||
|
|
+ self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
|
||
|
|
+
|
||
|
|
def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
|
||
|
|
self.m_readurl.side_effect = \
|
||
|
|
[SentinelException] * self.periodic_logging_attempts + \
|
||
|
|
--
|
||
|
|
2.27.0
|
||
|
|
|