From f11bbe7f04a48eebcb446e283820d7592f76cf86 Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Thu, 25 Mar 2021 07:20:10 -0700 Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between retries (#842) RH-Author: Eduardo Otubo RH-MergeRequest: 45: Add support for userdata on Azure from IMDS RH-Commit: [2/7] e8f8bb658b629a8444bd2ba19f109952acf33311 RH-Bugzilla: 2023940 RH-Acked-by: Emanuele Giuseppe Esposito RH-Acked-by: Mohamed Gamal Morsy Ensure that the Azure helper's http handler sleeps a fixed duration between retry failure attempts. The http handler will sleep a fixed duration between failed attempts regardless of whether the attempt failed due to (1) request timing out or (2) instant failure (no timeout). Due to certain platform issues, the http request to the Azure endpoint may instantly fail without reaching the http timeout duration. Without sleeping a fixed duration in between retry attempts, the http handler will loop through the max retry attempts quickly. This causes the communication between cloud-init and the Azure platform to be less resilient due to the short total duration if there is no sleep in between retries. --- cloudinit/sources/helpers/azure.py | 2 ++ tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index d3055d08..03e7156b 100755 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str: max_readurl_attempts = 240 default_readurl_timeout = 5 + sleep_duration_between_retries = 5 periodic_logging_attempts = 12 if 'timeout' not in kwargs: @@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str: 'attempt %d with exception: %s' % (url, attempt, e), logger_func=LOG.debug) + time.sleep(sleep_duration_between_retries) raise exc diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py index b8899807..63482c6c 100644 --- a/tests/unittests/test_datasource/test_azure_helper.py +++ b/tests/unittests/test_datasource/test_azure_helper.py @@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase): max_readurl_attempts = 240 default_readurl_timeout = 5 + sleep_duration_between_retries = 5 periodic_logging_attempts = 12 def setUp(self): @@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase): self.m_readurl = patches.enter_context( mock.patch.object( azure_helper.url_helper, 'readurl', mock.MagicMock())) - patches.enter_context( - mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) + self.m_sleep = patches.enter_context( + mock.patch.object(azure_helper.time, 'sleep', autospec=True)) def test_http_with_retries(self): self.m_readurl.return_value = 'TestResp' @@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase): self.m_readurl.call_count, self.periodic_logging_attempts + 1) + # Ensure that cloud-init did sleep between each failed request + self.assertEqual( + self.m_sleep.call_count, + self.periodic_logging_attempts) + self.m_sleep.assert_called_with(self.sleep_duration_between_retries) + def test_http_with_retries_long_delay_logs_periodic_failure_msg(self): self.m_readurl.side_effect = \ [SentinelException] * self.periodic_logging_attempts + \ -- 2.27.0