import cloud-init-21.1-17.el9
This commit is contained in:
parent
bd58381243
commit
125cbcb096
295
SOURCES/ci-Add-flexibility-to-IMDS-api-version-793.patch
Normal file
295
SOURCES/ci-Add-flexibility-to-IMDS-api-version-793.patch
Normal file
@ -0,0 +1,295 @@
|
||||
From f844e9c263e59a623ca8c647bd87bf4f91374d54 Mon Sep 17 00:00:00 2001
|
||||
From: Thomas Stringer <thstring@microsoft.com>
|
||||
Date: Wed, 3 Mar 2021 11:07:43 -0500
|
||||
Subject: [PATCH 1/7] Add flexibility to IMDS api-version (#793)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [1/7] 99a3db20e3f277a2f12ea21e937e06939434a2ca (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
Add flexibility to IMDS api-version by having both a desired IMDS
|
||||
api-version and a minimum api-version. The desired api-version will
|
||||
be used first, and if that fails it will fall back to the minimum
|
||||
api-version.
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 113 ++++++++++++++----
|
||||
tests/unittests/test_datasource/test_azure.py | 42 ++++++-
|
||||
2 files changed, 129 insertions(+), 26 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index 553b5a7e..de1452ce 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -78,17 +78,15 @@ AGENT_SEED_DIR = '/var/lib/waagent'
|
||||
# In the event where the IMDS primary server is not
|
||||
# available, it takes 1s to fallback to the secondary one
|
||||
IMDS_TIMEOUT_IN_SECONDS = 2
|
||||
-IMDS_URL = "http://169.254.169.254/metadata/"
|
||||
-IMDS_VER = "2019-06-01"
|
||||
-IMDS_VER_PARAM = "api-version={}".format(IMDS_VER)
|
||||
+IMDS_URL = "http://169.254.169.254/metadata"
|
||||
+IMDS_VER_MIN = "2019-06-01"
|
||||
+IMDS_VER_WANT = "2020-09-01"
|
||||
|
||||
|
||||
class metadata_type(Enum):
|
||||
- compute = "{}instance?{}".format(IMDS_URL, IMDS_VER_PARAM)
|
||||
- network = "{}instance/network?{}".format(IMDS_URL,
|
||||
- IMDS_VER_PARAM)
|
||||
- reprovisiondata = "{}reprovisiondata?{}".format(IMDS_URL,
|
||||
- IMDS_VER_PARAM)
|
||||
+ compute = "{}/instance".format(IMDS_URL)
|
||||
+ network = "{}/instance/network".format(IMDS_URL)
|
||||
+ reprovisiondata = "{}/reprovisiondata".format(IMDS_URL)
|
||||
|
||||
|
||||
PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0"
|
||||
@@ -349,6 +347,8 @@ class DataSourceAzure(sources.DataSource):
|
||||
self.update_events['network'].add(EventType.BOOT)
|
||||
self._ephemeral_dhcp_ctx = None
|
||||
|
||||
+ self.failed_desired_api_version = False
|
||||
+
|
||||
def __str__(self):
|
||||
root = sources.DataSource.__str__(self)
|
||||
return "%s [seed=%s]" % (root, self.seed)
|
||||
@@ -520,8 +520,10 @@ class DataSourceAzure(sources.DataSource):
|
||||
self._wait_for_all_nics_ready()
|
||||
ret = self._reprovision()
|
||||
|
||||
- imds_md = get_metadata_from_imds(
|
||||
- self.fallback_interface, retries=10)
|
||||
+ imds_md = self.get_imds_data_with_api_fallback(
|
||||
+ self.fallback_interface,
|
||||
+ retries=10
|
||||
+ )
|
||||
(md, userdata_raw, cfg, files) = ret
|
||||
self.seed = cdev
|
||||
crawled_data.update({
|
||||
@@ -652,6 +654,57 @@ class DataSourceAzure(sources.DataSource):
|
||||
self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700)
|
||||
return True
|
||||
|
||||
+ @azure_ds_telemetry_reporter
|
||||
+ def get_imds_data_with_api_fallback(
|
||||
+ self,
|
||||
+ fallback_nic,
|
||||
+ retries,
|
||||
+ md_type=metadata_type.compute):
|
||||
+ """
|
||||
+ Wrapper for get_metadata_from_imds so that we can have flexibility
|
||||
+ in which IMDS api-version we use. If a particular instance of IMDS
|
||||
+ does not have the api version that is desired, we want to make
|
||||
+ this fault tolerant and fall back to a good known minimum api
|
||||
+ version.
|
||||
+ """
|
||||
+
|
||||
+ if not self.failed_desired_api_version:
|
||||
+ for _ in range(retries):
|
||||
+ try:
|
||||
+ LOG.info(
|
||||
+ "Attempting IMDS api-version: %s",
|
||||
+ IMDS_VER_WANT
|
||||
+ )
|
||||
+ return get_metadata_from_imds(
|
||||
+ fallback_nic=fallback_nic,
|
||||
+ retries=0,
|
||||
+ md_type=md_type,
|
||||
+ api_version=IMDS_VER_WANT
|
||||
+ )
|
||||
+ except UrlError as err:
|
||||
+ LOG.info(
|
||||
+ "UrlError with IMDS api-version: %s",
|
||||
+ IMDS_VER_WANT
|
||||
+ )
|
||||
+ if err.code == 400:
|
||||
+ log_msg = "Fall back to IMDS api-version: {}".format(
|
||||
+ IMDS_VER_MIN
|
||||
+ )
|
||||
+ report_diagnostic_event(
|
||||
+ log_msg,
|
||||
+ logger_func=LOG.info
|
||||
+ )
|
||||
+ self.failed_desired_api_version = True
|
||||
+ break
|
||||
+
|
||||
+ LOG.info("Using IMDS api-version: %s", IMDS_VER_MIN)
|
||||
+ return get_metadata_from_imds(
|
||||
+ fallback_nic=fallback_nic,
|
||||
+ retries=retries,
|
||||
+ md_type=md_type,
|
||||
+ api_version=IMDS_VER_MIN
|
||||
+ )
|
||||
+
|
||||
def device_name_to_device(self, name):
|
||||
return self.ds_cfg['disk_aliases'].get(name)
|
||||
|
||||
@@ -880,10 +933,11 @@ class DataSourceAzure(sources.DataSource):
|
||||
# primary nic is being attached first helps here. Otherwise each nic
|
||||
# could add several seconds of delay.
|
||||
try:
|
||||
- imds_md = get_metadata_from_imds(
|
||||
+ imds_md = self.get_imds_data_with_api_fallback(
|
||||
ifname,
|
||||
5,
|
||||
- metadata_type.network)
|
||||
+ metadata_type.network
|
||||
+ )
|
||||
except Exception as e:
|
||||
LOG.warning(
|
||||
"Failed to get network metadata using nic %s. Attempt to "
|
||||
@@ -1017,7 +1071,10 @@ class DataSourceAzure(sources.DataSource):
|
||||
def _poll_imds(self):
|
||||
"""Poll IMDS for the new provisioning data until we get a valid
|
||||
response. Then return the returned JSON object."""
|
||||
- url = metadata_type.reprovisiondata.value
|
||||
+ url = "{}?api-version={}".format(
|
||||
+ metadata_type.reprovisiondata.value,
|
||||
+ IMDS_VER_MIN
|
||||
+ )
|
||||
headers = {"Metadata": "true"}
|
||||
nl_sock = None
|
||||
report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE))
|
||||
@@ -2059,7 +2116,8 @@ def _generate_network_config_from_fallback_config() -> dict:
|
||||
@azure_ds_telemetry_reporter
|
||||
def get_metadata_from_imds(fallback_nic,
|
||||
retries,
|
||||
- md_type=metadata_type.compute):
|
||||
+ md_type=metadata_type.compute,
|
||||
+ api_version=IMDS_VER_MIN):
|
||||
"""Query Azure's instance metadata service, returning a dictionary.
|
||||
|
||||
If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
|
||||
@@ -2069,13 +2127,16 @@ def get_metadata_from_imds(fallback_nic,
|
||||
@param fallback_nic: String. The name of the nic which requires active
|
||||
network in order to query IMDS.
|
||||
@param retries: The number of retries of the IMDS_URL.
|
||||
+ @param md_type: Metadata type for IMDS request.
|
||||
+ @param api_version: IMDS api-version to use in the request.
|
||||
|
||||
@return: A dict of instance metadata containing compute and network
|
||||
info.
|
||||
"""
|
||||
kwargs = {'logfunc': LOG.debug,
|
||||
'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
|
||||
- 'func': _get_metadata_from_imds, 'args': (retries, md_type,)}
|
||||
+ 'func': _get_metadata_from_imds,
|
||||
+ 'args': (retries, md_type, api_version,)}
|
||||
if net.is_up(fallback_nic):
|
||||
return util.log_time(**kwargs)
|
||||
else:
|
||||
@@ -2091,20 +2152,26 @@ def get_metadata_from_imds(fallback_nic,
|
||||
|
||||
|
||||
@azure_ds_telemetry_reporter
|
||||
-def _get_metadata_from_imds(retries, md_type=metadata_type.compute):
|
||||
-
|
||||
- url = md_type.value
|
||||
+def _get_metadata_from_imds(
|
||||
+ retries,
|
||||
+ md_type=metadata_type.compute,
|
||||
+ api_version=IMDS_VER_MIN):
|
||||
+ url = "{}?api-version={}".format(md_type.value, api_version)
|
||||
headers = {"Metadata": "true"}
|
||||
try:
|
||||
response = readurl(
|
||||
url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers,
|
||||
retries=retries, exception_cb=retry_on_url_exc)
|
||||
except Exception as e:
|
||||
- report_diagnostic_event(
|
||||
- 'Ignoring IMDS instance metadata. '
|
||||
- 'Get metadata from IMDS failed: %s' % e,
|
||||
- logger_func=LOG.warning)
|
||||
- return {}
|
||||
+ # pylint:disable=no-member
|
||||
+ if isinstance(e, UrlError) and e.code == 400:
|
||||
+ raise
|
||||
+ else:
|
||||
+ report_diagnostic_event(
|
||||
+ 'Ignoring IMDS instance metadata. '
|
||||
+ 'Get metadata from IMDS failed: %s' % e,
|
||||
+ logger_func=LOG.warning)
|
||||
+ return {}
|
||||
try:
|
||||
from json.decoder import JSONDecodeError
|
||||
json_decode_error = JSONDecodeError
|
||||
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
|
||||
index f597c723..dedebeb1 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure.py
|
||||
@@ -408,7 +408,9 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestGetMetadataFromIMDS, self).setUp()
|
||||
- self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01"
|
||||
+ self.network_md_url = "{}/instance?api-version=2019-06-01".format(
|
||||
+ dsaz.IMDS_URL
|
||||
+ )
|
||||
|
||||
@mock.patch(MOCKPATH + 'readurl')
|
||||
@mock.patch(MOCKPATH + 'EphemeralDHCPv4', autospec=True)
|
||||
@@ -518,7 +520,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
"""Return empty dict when IMDS network metadata is absent."""
|
||||
httpretty.register_uri(
|
||||
httpretty.GET,
|
||||
- dsaz.IMDS_URL + 'instance?api-version=2017-12-01',
|
||||
+ dsaz.IMDS_URL + '/instance?api-version=2017-12-01',
|
||||
body={}, status=404)
|
||||
|
||||
m_net_is_up.return_value = True # skips dhcp
|
||||
@@ -1877,6 +1879,40 @@ scbus-1 on xpt0 bus 0
|
||||
ssh_keys = dsrc.get_public_ssh_keys()
|
||||
self.assertEqual(ssh_keys, ['key2'])
|
||||
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_imds_api_version_wanted_nonexistent(
|
||||
+ self,
|
||||
+ m_get_metadata_from_imds):
|
||||
+ def get_metadata_from_imds_side_eff(*args, **kwargs):
|
||||
+ if kwargs['api_version'] == dsaz.IMDS_VER_WANT:
|
||||
+ raise url_helper.UrlError("No IMDS version", code=400)
|
||||
+ return NETWORK_METADATA
|
||||
+ m_get_metadata_from_imds.side_effect = get_metadata_from_imds_side_eff
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ self.assertIsNotNone(dsrc.metadata)
|
||||
+ self.assertTrue(dsrc.failed_desired_api_version)
|
||||
+
|
||||
+ @mock.patch(
|
||||
+ MOCKPATH + 'get_metadata_from_imds', return_value=NETWORK_METADATA)
|
||||
+ def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds):
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ self.assertIsNotNone(dsrc.metadata)
|
||||
+ self.assertFalse(dsrc.failed_desired_api_version)
|
||||
+
|
||||
|
||||
class TestAzureBounce(CiTestCase):
|
||||
|
||||
@@ -2657,7 +2693,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
|
||||
@mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up')
|
||||
@mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event')
|
||||
@mock.patch('cloudinit.sources.net.find_fallback_nic')
|
||||
- @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback')
|
||||
@mock.patch(MOCKPATH + 'EphemeralDHCPv4')
|
||||
@mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach')
|
||||
@mock.patch('os.path.isfile')
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,397 @@
|
||||
From 68f058e8d20a499f74bc78af8e0c6a90ca57ae20 Mon Sep 17 00:00:00 2001
|
||||
From: Thomas Stringer <thstring@microsoft.com>
|
||||
Date: Mon, 26 Apr 2021 09:41:38 -0400
|
||||
Subject: [PATCH 5/7] Azure: Retrieve username and hostname from IMDS (#865)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [5/7] 6a768d31e63e5f00dae0fad2712a7618d62b0879 (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
This change allows us to retrieve the username and hostname from
|
||||
IMDS instead of having to rely on the mounted OVF.
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 149 ++++++++++++++----
|
||||
tests/unittests/test_datasource/test_azure.py | 87 +++++++++-
|
||||
2 files changed, 205 insertions(+), 31 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index 39e67c4f..6d7954ee 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -5,6 +5,7 @@
|
||||
# This file is part of cloud-init. See LICENSE file for license information.
|
||||
|
||||
import base64
|
||||
+from collections import namedtuple
|
||||
import contextlib
|
||||
import crypt
|
||||
from functools import partial
|
||||
@@ -25,6 +26,7 @@ from cloudinit.net import device_driver
|
||||
from cloudinit.net.dhcp import EphemeralDHCPv4
|
||||
from cloudinit import sources
|
||||
from cloudinit.sources.helpers import netlink
|
||||
+from cloudinit import ssh_util
|
||||
from cloudinit import subp
|
||||
from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc
|
||||
from cloudinit import util
|
||||
@@ -80,7 +82,12 @@ AGENT_SEED_DIR = '/var/lib/waagent'
|
||||
IMDS_TIMEOUT_IN_SECONDS = 2
|
||||
IMDS_URL = "http://169.254.169.254/metadata"
|
||||
IMDS_VER_MIN = "2019-06-01"
|
||||
-IMDS_VER_WANT = "2020-09-01"
|
||||
+IMDS_VER_WANT = "2020-10-01"
|
||||
+
|
||||
+
|
||||
+# This holds SSH key data including if the source was
|
||||
+# from IMDS, as well as the SSH key data itself.
|
||||
+SSHKeys = namedtuple("SSHKeys", ("keys_from_imds", "ssh_keys"))
|
||||
|
||||
|
||||
class metadata_type(Enum):
|
||||
@@ -391,6 +398,8 @@ class DataSourceAzure(sources.DataSource):
|
||||
"""Return the subplatform metadata source details."""
|
||||
if self.seed.startswith('/dev'):
|
||||
subplatform_type = 'config-disk'
|
||||
+ elif self.seed.lower() == 'imds':
|
||||
+ subplatform_type = 'imds'
|
||||
else:
|
||||
subplatform_type = 'seed-dir'
|
||||
return '%s (%s)' % (subplatform_type, self.seed)
|
||||
@@ -433,9 +442,11 @@ class DataSourceAzure(sources.DataSource):
|
||||
|
||||
found = None
|
||||
reprovision = False
|
||||
+ ovf_is_accessible = True
|
||||
reprovision_after_nic_attach = False
|
||||
for cdev in candidates:
|
||||
try:
|
||||
+ LOG.debug("cdev: %s", cdev)
|
||||
if cdev == "IMDS":
|
||||
ret = None
|
||||
reprovision = True
|
||||
@@ -462,8 +473,18 @@ class DataSourceAzure(sources.DataSource):
|
||||
raise sources.InvalidMetaDataException(msg)
|
||||
except util.MountFailedError:
|
||||
report_diagnostic_event(
|
||||
- '%s was not mountable' % cdev, logger_func=LOG.warning)
|
||||
- continue
|
||||
+ '%s was not mountable' % cdev, logger_func=LOG.debug)
|
||||
+ cdev = 'IMDS'
|
||||
+ ovf_is_accessible = False
|
||||
+ empty_md = {'local-hostname': ''}
|
||||
+ empty_cfg = dict(
|
||||
+ system_info=dict(
|
||||
+ default_user=dict(
|
||||
+ name=''
|
||||
+ )
|
||||
+ )
|
||||
+ )
|
||||
+ ret = (empty_md, '', empty_cfg, {})
|
||||
|
||||
report_diagnostic_event("Found provisioning metadata in %s" % cdev,
|
||||
logger_func=LOG.debug)
|
||||
@@ -490,6 +511,10 @@ class DataSourceAzure(sources.DataSource):
|
||||
self.fallback_interface,
|
||||
retries=10
|
||||
)
|
||||
+ if not imds_md and not ovf_is_accessible:
|
||||
+ msg = 'No OVF or IMDS available'
|
||||
+ report_diagnostic_event(msg)
|
||||
+ raise sources.InvalidMetaDataException(msg)
|
||||
(md, userdata_raw, cfg, files) = ret
|
||||
self.seed = cdev
|
||||
crawled_data.update({
|
||||
@@ -498,6 +523,21 @@ class DataSourceAzure(sources.DataSource):
|
||||
'metadata': util.mergemanydict(
|
||||
[md, {'imds': imds_md}]),
|
||||
'userdata_raw': userdata_raw})
|
||||
+ imds_username = _username_from_imds(imds_md)
|
||||
+ imds_hostname = _hostname_from_imds(imds_md)
|
||||
+ imds_disable_password = _disable_password_from_imds(imds_md)
|
||||
+ if imds_username:
|
||||
+ LOG.debug('Username retrieved from IMDS: %s', imds_username)
|
||||
+ cfg['system_info']['default_user']['name'] = imds_username
|
||||
+ if imds_hostname:
|
||||
+ LOG.debug('Hostname retrieved from IMDS: %s', imds_hostname)
|
||||
+ crawled_data['metadata']['local-hostname'] = imds_hostname
|
||||
+ if imds_disable_password:
|
||||
+ LOG.debug(
|
||||
+ 'Disable password retrieved from IMDS: %s',
|
||||
+ imds_disable_password
|
||||
+ )
|
||||
+ crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501
|
||||
found = cdev
|
||||
|
||||
report_diagnostic_event(
|
||||
@@ -676,6 +716,13 @@ class DataSourceAzure(sources.DataSource):
|
||||
|
||||
@azure_ds_telemetry_reporter
|
||||
def get_public_ssh_keys(self):
|
||||
+ """
|
||||
+ Retrieve public SSH keys.
|
||||
+ """
|
||||
+
|
||||
+ return self._get_public_ssh_keys_and_source().ssh_keys
|
||||
+
|
||||
+ def _get_public_ssh_keys_and_source(self):
|
||||
"""
|
||||
Try to get the ssh keys from IMDS first, and if that fails
|
||||
(i.e. IMDS is unavailable) then fallback to getting the ssh
|
||||
@@ -685,30 +732,50 @@ class DataSourceAzure(sources.DataSource):
|
||||
advantage, so this is a strong preference. But we must keep
|
||||
OVF as a second option for environments that don't have IMDS.
|
||||
"""
|
||||
+
|
||||
LOG.debug('Retrieving public SSH keys')
|
||||
ssh_keys = []
|
||||
+ keys_from_imds = True
|
||||
+ LOG.debug('Attempting to get SSH keys from IMDS')
|
||||
try:
|
||||
- raise KeyError(
|
||||
- "Not using public SSH keys from IMDS"
|
||||
- )
|
||||
- # pylint:disable=unreachable
|
||||
ssh_keys = [
|
||||
public_key['keyData']
|
||||
for public_key
|
||||
in self.metadata['imds']['compute']['publicKeys']
|
||||
]
|
||||
- LOG.debug('Retrieved SSH keys from IMDS')
|
||||
+ for key in ssh_keys:
|
||||
+ if not _key_is_openssh_formatted(key=key):
|
||||
+ keys_from_imds = False
|
||||
+ break
|
||||
+
|
||||
+ if not keys_from_imds:
|
||||
+ log_msg = 'Keys not in OpenSSH format, using OVF'
|
||||
+ else:
|
||||
+ log_msg = 'Retrieved {} keys from IMDS'.format(
|
||||
+ len(ssh_keys)
|
||||
+ if ssh_keys is not None
|
||||
+ else 0
|
||||
+ )
|
||||
except KeyError:
|
||||
log_msg = 'Unable to get keys from IMDS, falling back to OVF'
|
||||
+ keys_from_imds = False
|
||||
+ finally:
|
||||
report_diagnostic_event(log_msg, logger_func=LOG.debug)
|
||||
+
|
||||
+ if not keys_from_imds:
|
||||
+ LOG.debug('Attempting to get SSH keys from OVF')
|
||||
try:
|
||||
ssh_keys = self.metadata['public-keys']
|
||||
- LOG.debug('Retrieved keys from OVF')
|
||||
+ log_msg = 'Retrieved {} keys from OVF'.format(len(ssh_keys))
|
||||
except KeyError:
|
||||
log_msg = 'No keys available from OVF'
|
||||
+ finally:
|
||||
report_diagnostic_event(log_msg, logger_func=LOG.debug)
|
||||
|
||||
- return ssh_keys
|
||||
+ return SSHKeys(
|
||||
+ keys_from_imds=keys_from_imds,
|
||||
+ ssh_keys=ssh_keys
|
||||
+ )
|
||||
|
||||
def get_config_obj(self):
|
||||
return self.cfg
|
||||
@@ -1325,30 +1392,21 @@ class DataSourceAzure(sources.DataSource):
|
||||
self.bounce_network_with_azure_hostname()
|
||||
|
||||
pubkey_info = None
|
||||
- try:
|
||||
- raise KeyError(
|
||||
- "Not using public SSH keys from IMDS"
|
||||
- )
|
||||
- # pylint:disable=unreachable
|
||||
- public_keys = self.metadata['imds']['compute']['publicKeys']
|
||||
- LOG.debug(
|
||||
- 'Successfully retrieved %s key(s) from IMDS',
|
||||
- len(public_keys)
|
||||
- if public_keys is not None
|
||||
+ ssh_keys_and_source = self._get_public_ssh_keys_and_source()
|
||||
+
|
||||
+ if not ssh_keys_and_source.keys_from_imds:
|
||||
+ pubkey_info = self.cfg.get('_pubkeys', None)
|
||||
+ log_msg = 'Retrieved {} fingerprints from OVF'.format(
|
||||
+ len(pubkey_info)
|
||||
+ if pubkey_info is not None
|
||||
else 0
|
||||
)
|
||||
- except KeyError:
|
||||
- LOG.debug(
|
||||
- 'Unable to retrieve SSH keys from IMDS during '
|
||||
- 'negotiation, falling back to OVF'
|
||||
- )
|
||||
- pubkey_info = self.cfg.get('_pubkeys', None)
|
||||
+ report_diagnostic_event(log_msg, logger_func=LOG.debug)
|
||||
|
||||
metadata_func = partial(get_metadata_from_fabric,
|
||||
fallback_lease_file=self.
|
||||
dhclient_lease_file,
|
||||
- pubkey_info=pubkey_info,
|
||||
- iso_dev=self.iso_dev)
|
||||
+ pubkey_info=pubkey_info)
|
||||
|
||||
LOG.debug("negotiating with fabric via agent command %s",
|
||||
self.ds_cfg['agent_command'])
|
||||
@@ -1404,6 +1462,41 @@ class DataSourceAzure(sources.DataSource):
|
||||
return self.metadata.get('imds', {}).get('compute', {}).get('location')
|
||||
|
||||
|
||||
+def _username_from_imds(imds_data):
|
||||
+ try:
|
||||
+ return imds_data['compute']['osProfile']['adminUsername']
|
||||
+ except KeyError:
|
||||
+ return None
|
||||
+
|
||||
+
|
||||
+def _hostname_from_imds(imds_data):
|
||||
+ try:
|
||||
+ return imds_data['compute']['osProfile']['computerName']
|
||||
+ except KeyError:
|
||||
+ return None
|
||||
+
|
||||
+
|
||||
+def _disable_password_from_imds(imds_data):
|
||||
+ try:
|
||||
+ return imds_data['compute']['osProfile']['disablePasswordAuthentication'] == 'true' # noqa: E501
|
||||
+ except KeyError:
|
||||
+ return None
|
||||
+
|
||||
+
|
||||
+def _key_is_openssh_formatted(key):
|
||||
+ """
|
||||
+ Validate whether or not the key is OpenSSH-formatted.
|
||||
+ """
|
||||
+
|
||||
+ parser = ssh_util.AuthKeyLineParser()
|
||||
+ try:
|
||||
+ akl = parser.parse(key)
|
||||
+ except TypeError:
|
||||
+ return False
|
||||
+
|
||||
+ return akl.keytype is not None
|
||||
+
|
||||
+
|
||||
def _partitions_on_device(devpath, maxnum=16):
|
||||
# return a list of tuples (ptnum, path) for each part on devpath
|
||||
for suff in ("-part", "p", ""):
|
||||
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
|
||||
index 320fa857..d9817d84 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure.py
|
||||
@@ -108,7 +108,7 @@ NETWORK_METADATA = {
|
||||
"zone": "",
|
||||
"publicKeys": [
|
||||
{
|
||||
- "keyData": "key1",
|
||||
+ "keyData": "ssh-rsa key1",
|
||||
"path": "path1"
|
||||
}
|
||||
]
|
||||
@@ -1761,8 +1761,29 @@ scbus-1 on xpt0 bus 0
|
||||
dsrc.get_data()
|
||||
dsrc.setup(True)
|
||||
ssh_keys = dsrc.get_public_ssh_keys()
|
||||
- # Temporarily alter this test so that SSH public keys
|
||||
- # from IMDS are *not* going to be in use to fix a regression.
|
||||
+ self.assertEqual(ssh_keys, ["ssh-rsa key1"])
|
||||
+ self.assertEqual(m_parse_certificates.call_count, 0)
|
||||
+
|
||||
+ @mock.patch(
|
||||
+ 'cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates')
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_get_public_ssh_keys_with_no_openssh_format(
|
||||
+ self,
|
||||
+ m_get_metadata_from_imds,
|
||||
+ m_parse_certificates):
|
||||
+ imds_data = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data['compute']['publicKeys'][0]['keyData'] = 'no-openssh-format'
|
||||
+ m_get_metadata_from_imds.return_value = imds_data
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ dsrc.setup(True)
|
||||
+ ssh_keys = dsrc.get_public_ssh_keys()
|
||||
self.assertEqual(ssh_keys, [])
|
||||
self.assertEqual(m_parse_certificates.call_count, 0)
|
||||
|
||||
@@ -1818,6 +1839,66 @@ scbus-1 on xpt0 bus 0
|
||||
self.assertIsNotNone(dsrc.metadata)
|
||||
self.assertFalse(dsrc.failed_desired_api_version)
|
||||
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_hostname_from_imds(self, m_get_metadata_from_imds):
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data_with_os_profile["compute"]["osProfile"] = dict(
|
||||
+ adminUsername="username1",
|
||||
+ computerName="hostname1",
|
||||
+ disablePasswordAuthentication="true"
|
||||
+ )
|
||||
+ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ self.assertEqual(dsrc.metadata["local-hostname"], "hostname1")
|
||||
+
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_username_from_imds(self, m_get_metadata_from_imds):
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data_with_os_profile["compute"]["osProfile"] = dict(
|
||||
+ adminUsername="username1",
|
||||
+ computerName="hostname1",
|
||||
+ disablePasswordAuthentication="true"
|
||||
+ )
|
||||
+ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ self.assertEqual(
|
||||
+ dsrc.cfg["system_info"]["default_user"]["name"],
|
||||
+ "username1"
|
||||
+ )
|
||||
+
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_disable_password_from_imds(self, m_get_metadata_from_imds):
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data_with_os_profile["compute"]["osProfile"] = dict(
|
||||
+ adminUsername="username1",
|
||||
+ computerName="hostname1",
|
||||
+ disablePasswordAuthentication="true"
|
||||
+ )
|
||||
+ m_get_metadata_from_imds.return_value = imds_data_with_os_profile
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ dsrc.get_data()
|
||||
+ self.assertTrue(dsrc.metadata["disable_password"])
|
||||
+
|
||||
|
||||
class TestAzureBounce(CiTestCase):
|
||||
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,315 @@
|
||||
From 816fe5c2e6d5dcc68f292092b00b2acfbc4c8e88 Mon Sep 17 00:00:00 2001
|
||||
From: aswinrajamannar <39812128+aswinrajamannar@users.noreply.github.com>
|
||||
Date: Mon, 26 Apr 2021 07:28:39 -0700
|
||||
Subject: [PATCH 6/7] Azure: Retry net metadata during nic attach for
|
||||
non-timeout errs (#878)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [6/7] 794cd340644260bb43a7c8582a8067f403b9842d (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
When network interfaces are hot-attached to the VM, attempting to get
|
||||
network metadata might return 410 (or 500, 503 etc) because the info
|
||||
is not yet available. In those cases, we retry getting the metadata
|
||||
before giving up. The only case where we can move on to wait for more
|
||||
nic attach events is if the call times out despite retries, which
|
||||
means the interface is not likely a primary interface, and we should
|
||||
try for more nic attach events.
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 65 +++++++++++--
|
||||
tests/unittests/test_datasource/test_azure.py | 95 ++++++++++++++++---
|
||||
2 files changed, 140 insertions(+), 20 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index 6d7954ee..d0be6d84 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -17,6 +17,7 @@ from time import sleep
|
||||
from xml.dom import minidom
|
||||
import xml.etree.ElementTree as ET
|
||||
from enum import Enum
|
||||
+import requests
|
||||
|
||||
from cloudinit import dmi
|
||||
from cloudinit import log as logging
|
||||
@@ -665,7 +666,9 @@ class DataSourceAzure(sources.DataSource):
|
||||
self,
|
||||
fallback_nic,
|
||||
retries,
|
||||
- md_type=metadata_type.compute):
|
||||
+ md_type=metadata_type.compute,
|
||||
+ exc_cb=retry_on_url_exc,
|
||||
+ infinite=False):
|
||||
"""
|
||||
Wrapper for get_metadata_from_imds so that we can have flexibility
|
||||
in which IMDS api-version we use. If a particular instance of IMDS
|
||||
@@ -685,7 +688,8 @@ class DataSourceAzure(sources.DataSource):
|
||||
fallback_nic=fallback_nic,
|
||||
retries=0,
|
||||
md_type=md_type,
|
||||
- api_version=IMDS_VER_WANT
|
||||
+ api_version=IMDS_VER_WANT,
|
||||
+ exc_cb=exc_cb
|
||||
)
|
||||
except UrlError as err:
|
||||
LOG.info(
|
||||
@@ -708,7 +712,9 @@ class DataSourceAzure(sources.DataSource):
|
||||
fallback_nic=fallback_nic,
|
||||
retries=retries,
|
||||
md_type=md_type,
|
||||
- api_version=IMDS_VER_MIN
|
||||
+ api_version=IMDS_VER_MIN,
|
||||
+ exc_cb=exc_cb,
|
||||
+ infinite=infinite
|
||||
)
|
||||
|
||||
def device_name_to_device(self, name):
|
||||
@@ -938,6 +944,9 @@ class DataSourceAzure(sources.DataSource):
|
||||
is_primary = False
|
||||
expected_nic_count = -1
|
||||
imds_md = None
|
||||
+ metadata_poll_count = 0
|
||||
+ metadata_logging_threshold = 1
|
||||
+ metadata_timeout_count = 0
|
||||
|
||||
# For now, only a VM's primary NIC can contact IMDS and WireServer. If
|
||||
# DHCP fails for a NIC, we have no mechanism to determine if the NIC is
|
||||
@@ -962,14 +971,48 @@ class DataSourceAzure(sources.DataSource):
|
||||
% (ifname, e), logger_func=LOG.error)
|
||||
raise
|
||||
|
||||
+ # Retry polling network metadata for a limited duration only when the
|
||||
+ # calls fail due to timeout. This is because the platform drops packets
|
||||
+ # going towards IMDS when it is not a primary nic. If the calls fail
|
||||
+ # due to other issues like 410, 503 etc, then it means we are primary
|
||||
+ # but IMDS service is unavailable at the moment. Retry indefinitely in
|
||||
+ # those cases since we cannot move on without the network metadata.
|
||||
+ def network_metadata_exc_cb(msg, exc):
|
||||
+ nonlocal metadata_timeout_count, metadata_poll_count
|
||||
+ nonlocal metadata_logging_threshold
|
||||
+
|
||||
+ metadata_poll_count = metadata_poll_count + 1
|
||||
+
|
||||
+ # Log when needed but back off exponentially to avoid exploding
|
||||
+ # the log file.
|
||||
+ if metadata_poll_count >= metadata_logging_threshold:
|
||||
+ metadata_logging_threshold *= 2
|
||||
+ report_diagnostic_event(
|
||||
+ "Ran into exception when attempting to reach %s "
|
||||
+ "after %d polls." % (msg, metadata_poll_count),
|
||||
+ logger_func=LOG.error)
|
||||
+
|
||||
+ if isinstance(exc, UrlError):
|
||||
+ report_diagnostic_event("poll IMDS with %s failed. "
|
||||
+ "Exception: %s and code: %s" %
|
||||
+ (msg, exc.cause, exc.code),
|
||||
+ logger_func=LOG.error)
|
||||
+
|
||||
+ if exc.cause and isinstance(exc.cause, requests.Timeout):
|
||||
+ metadata_timeout_count = metadata_timeout_count + 1
|
||||
+ return (metadata_timeout_count <= 10)
|
||||
+ return True
|
||||
+
|
||||
# Primary nic detection will be optimized in the future. The fact that
|
||||
# primary nic is being attached first helps here. Otherwise each nic
|
||||
# could add several seconds of delay.
|
||||
try:
|
||||
imds_md = self.get_imds_data_with_api_fallback(
|
||||
ifname,
|
||||
- 5,
|
||||
- metadata_type.network
|
||||
+ 0,
|
||||
+ metadata_type.network,
|
||||
+ network_metadata_exc_cb,
|
||||
+ True
|
||||
)
|
||||
except Exception as e:
|
||||
LOG.warning(
|
||||
@@ -2139,7 +2182,9 @@ def _generate_network_config_from_fallback_config() -> dict:
|
||||
def get_metadata_from_imds(fallback_nic,
|
||||
retries,
|
||||
md_type=metadata_type.compute,
|
||||
- api_version=IMDS_VER_MIN):
|
||||
+ api_version=IMDS_VER_MIN,
|
||||
+ exc_cb=retry_on_url_exc,
|
||||
+ infinite=False):
|
||||
"""Query Azure's instance metadata service, returning a dictionary.
|
||||
|
||||
If network is not up, setup ephemeral dhcp on fallback_nic to talk to the
|
||||
@@ -2158,7 +2203,7 @@ def get_metadata_from_imds(fallback_nic,
|
||||
kwargs = {'logfunc': LOG.debug,
|
||||
'msg': 'Crawl of Azure Instance Metadata Service (IMDS)',
|
||||
'func': _get_metadata_from_imds,
|
||||
- 'args': (retries, md_type, api_version,)}
|
||||
+ 'args': (retries, exc_cb, md_type, api_version, infinite)}
|
||||
if net.is_up(fallback_nic):
|
||||
return util.log_time(**kwargs)
|
||||
else:
|
||||
@@ -2176,14 +2221,16 @@ def get_metadata_from_imds(fallback_nic,
|
||||
@azure_ds_telemetry_reporter
|
||||
def _get_metadata_from_imds(
|
||||
retries,
|
||||
+ exc_cb,
|
||||
md_type=metadata_type.compute,
|
||||
- api_version=IMDS_VER_MIN):
|
||||
+ api_version=IMDS_VER_MIN,
|
||||
+ infinite=False):
|
||||
url = "{}?api-version={}".format(md_type.value, api_version)
|
||||
headers = {"Metadata": "true"}
|
||||
try:
|
||||
response = readurl(
|
||||
url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers,
|
||||
- retries=retries, exception_cb=retry_on_url_exc)
|
||||
+ retries=retries, exception_cb=exc_cb, infinite=infinite)
|
||||
except Exception as e:
|
||||
# pylint:disable=no-member
|
||||
if isinstance(e, UrlError) and e.code == 400:
|
||||
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
|
||||
index d9817d84..c4a8e08d 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure.py
|
||||
@@ -448,7 +448,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
"http://169.254.169.254/metadata/instance?api-version="
|
||||
"2019-06-01", exception_cb=mock.ANY,
|
||||
headers=mock.ANY, retries=mock.ANY,
|
||||
- timeout=mock.ANY)
|
||||
+ timeout=mock.ANY, infinite=False)
|
||||
|
||||
@mock.patch(MOCKPATH + 'readurl', autospec=True)
|
||||
@mock.patch(MOCKPATH + 'EphemeralDHCPv4')
|
||||
@@ -467,7 +467,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
"http://169.254.169.254/metadata/instance/network?api-version="
|
||||
"2019-06-01", exception_cb=mock.ANY,
|
||||
headers=mock.ANY, retries=mock.ANY,
|
||||
- timeout=mock.ANY)
|
||||
+ timeout=mock.ANY, infinite=False)
|
||||
|
||||
@mock.patch(MOCKPATH + 'readurl', autospec=True)
|
||||
@mock.patch(MOCKPATH + 'EphemeralDHCPv4')
|
||||
@@ -486,7 +486,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
"http://169.254.169.254/metadata/instance?api-version="
|
||||
"2019-06-01", exception_cb=mock.ANY,
|
||||
headers=mock.ANY, retries=mock.ANY,
|
||||
- timeout=mock.ANY)
|
||||
+ timeout=mock.ANY, infinite=False)
|
||||
|
||||
@mock.patch(MOCKPATH + 'readurl', autospec=True)
|
||||
@mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting', autospec=True)
|
||||
@@ -511,7 +511,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase):
|
||||
m_readurl.assert_called_with(
|
||||
self.network_md_url, exception_cb=mock.ANY,
|
||||
headers={'Metadata': 'true'}, retries=2,
|
||||
- timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS)
|
||||
+ timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, infinite=False)
|
||||
|
||||
@mock.patch('cloudinit.url_helper.time.sleep')
|
||||
@mock.patch(MOCKPATH + 'net.is_up', autospec=True)
|
||||
@@ -2694,15 +2694,22 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
|
||||
|
||||
def nic_attach_ret(nl_sock, nics_found):
|
||||
nonlocal m_attach_call_count
|
||||
- if m_attach_call_count == 0:
|
||||
- m_attach_call_count = m_attach_call_count + 1
|
||||
+ m_attach_call_count = m_attach_call_count + 1
|
||||
+ if m_attach_call_count == 1:
|
||||
return "eth0"
|
||||
- return "eth1"
|
||||
+ elif m_attach_call_count == 2:
|
||||
+ return "eth1"
|
||||
+ raise RuntimeError("Must have found primary nic by now.")
|
||||
+
|
||||
+ # Simulate two NICs by adding the same one twice.
|
||||
+ md = {
|
||||
+ "interface": [
|
||||
+ IMDS_NETWORK_METADATA['interface'][0],
|
||||
+ IMDS_NETWORK_METADATA['interface'][0]
|
||||
+ ]
|
||||
+ }
|
||||
|
||||
- def network_metadata_ret(ifname, retries, type):
|
||||
- # Simulate two NICs by adding the same one twice.
|
||||
- md = IMDS_NETWORK_METADATA
|
||||
- md['interface'].append(md['interface'][0])
|
||||
+ def network_metadata_ret(ifname, retries, type, exc_cb, infinite):
|
||||
if ifname == "eth0":
|
||||
return md
|
||||
raise requests.Timeout('Fake connection timeout')
|
||||
@@ -2724,6 +2731,72 @@ class TestPreprovisioningHotAttachNics(CiTestCase):
|
||||
self.assertEqual(1, m_imds.call_count)
|
||||
self.assertEqual(2, m_link_up.call_count)
|
||||
|
||||
+ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback')
|
||||
+ @mock.patch(MOCKPATH + 'EphemeralDHCPv4')
|
||||
+ def test_check_if_nic_is_primary_retries_on_failures(
|
||||
+ self, m_dhcpv4, m_imds):
|
||||
+ """Retry polling for network metadata on all failures except timeout"""
|
||||
+ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths)
|
||||
+ lease = {
|
||||
+ 'interface': 'eth9', 'fixed-address': '192.168.2.9',
|
||||
+ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0',
|
||||
+ 'unknown-245': '624c3620'}
|
||||
+
|
||||
+ eth0Retries = []
|
||||
+ eth1Retries = []
|
||||
+ # Simulate two NICs by adding the same one twice.
|
||||
+ md = {
|
||||
+ "interface": [
|
||||
+ IMDS_NETWORK_METADATA['interface'][0],
|
||||
+ IMDS_NETWORK_METADATA['interface'][0]
|
||||
+ ]
|
||||
+ }
|
||||
+
|
||||
+ def network_metadata_ret(ifname, retries, type, exc_cb, infinite):
|
||||
+ nonlocal eth0Retries, eth1Retries
|
||||
+
|
||||
+ # Simulate readurl functionality with retries and
|
||||
+ # exception callbacks so that the callback logic can be
|
||||
+ # validated.
|
||||
+ if ifname == "eth0":
|
||||
+ cause = requests.HTTPError()
|
||||
+ for _ in range(0, 15):
|
||||
+ error = url_helper.UrlError(cause=cause, code=410)
|
||||
+ eth0Retries.append(exc_cb("No goal state.", error))
|
||||
+ else:
|
||||
+ cause = requests.Timeout('Fake connection timeout')
|
||||
+ for _ in range(0, 10):
|
||||
+ error = url_helper.UrlError(cause=cause)
|
||||
+ eth1Retries.append(exc_cb("Connection timeout", error))
|
||||
+ # Should stop retrying after 10 retries
|
||||
+ eth1Retries.append(exc_cb("Connection timeout", error))
|
||||
+ raise cause
|
||||
+ return md
|
||||
+
|
||||
+ m_imds.side_effect = network_metadata_ret
|
||||
+
|
||||
+ dhcp_ctx = mock.MagicMock(lease=lease)
|
||||
+ dhcp_ctx.obtain_lease.return_value = lease
|
||||
+ m_dhcpv4.return_value = dhcp_ctx
|
||||
+
|
||||
+ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth0")
|
||||
+ self.assertEqual(True, is_primary)
|
||||
+ self.assertEqual(2, expected_nic_count)
|
||||
+
|
||||
+ # All Eth0 errors are non-timeout errors. So we should have been
|
||||
+ # retrying indefinitely until success.
|
||||
+ for i in eth0Retries:
|
||||
+ self.assertTrue(i)
|
||||
+
|
||||
+ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth1")
|
||||
+ self.assertEqual(False, is_primary)
|
||||
+
|
||||
+ # All Eth1 errors are timeout errors. Retry happens for a max of 10 and
|
||||
+ # then we should have moved on assuming it is not the primary nic.
|
||||
+ for i in range(0, 10):
|
||||
+ self.assertTrue(eth1Retries[i])
|
||||
+ self.assertFalse(eth1Retries[10])
|
||||
+
|
||||
@mock.patch('cloudinit.distros.networking.LinuxNetworking.try_set_link_up')
|
||||
def test_wait_for_link_up_returns_if_already_up(
|
||||
self, m_is_link_up):
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,129 @@
|
||||
From 0def71378dc7abf682727c600b696f7313cdcf60 Mon Sep 17 00:00:00 2001
|
||||
From: Anh Vo <anhvo@microsoft.com>
|
||||
Date: Tue, 27 Apr 2021 13:40:59 -0400
|
||||
Subject: [PATCH 7/7] Azure: adding support for consuming userdata from IMDS
|
||||
(#884)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [7/7] 1e7ab925162ed9ef2c9b5b9f5c6d5e6ec6e623dd (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 23 ++++++++-
|
||||
tests/unittests/test_datasource/test_azure.py | 50 +++++++++++++++++++
|
||||
2 files changed, 72 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index d0be6d84..a66f023d 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -83,7 +83,7 @@ AGENT_SEED_DIR = '/var/lib/waagent'
|
||||
IMDS_TIMEOUT_IN_SECONDS = 2
|
||||
IMDS_URL = "http://169.254.169.254/metadata"
|
||||
IMDS_VER_MIN = "2019-06-01"
|
||||
-IMDS_VER_WANT = "2020-10-01"
|
||||
+IMDS_VER_WANT = "2021-01-01"
|
||||
|
||||
|
||||
# This holds SSH key data including if the source was
|
||||
@@ -539,6 +539,20 @@ class DataSourceAzure(sources.DataSource):
|
||||
imds_disable_password
|
||||
)
|
||||
crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501
|
||||
+
|
||||
+ # only use userdata from imds if OVF did not provide custom data
|
||||
+ # userdata provided by IMDS is always base64 encoded
|
||||
+ if not userdata_raw:
|
||||
+ imds_userdata = _userdata_from_imds(imds_md)
|
||||
+ if imds_userdata:
|
||||
+ LOG.debug("Retrieved userdata from IMDS")
|
||||
+ try:
|
||||
+ crawled_data['userdata_raw'] = base64.b64decode(
|
||||
+ ''.join(imds_userdata.split()))
|
||||
+ except Exception:
|
||||
+ report_diagnostic_event(
|
||||
+ "Bad userdata in IMDS",
|
||||
+ logger_func=LOG.warning)
|
||||
found = cdev
|
||||
|
||||
report_diagnostic_event(
|
||||
@@ -1512,6 +1526,13 @@ def _username_from_imds(imds_data):
|
||||
return None
|
||||
|
||||
|
||||
+def _userdata_from_imds(imds_data):
|
||||
+ try:
|
||||
+ return imds_data['compute']['userData']
|
||||
+ except KeyError:
|
||||
+ return None
|
||||
+
|
||||
+
|
||||
def _hostname_from_imds(imds_data):
|
||||
try:
|
||||
return imds_data['compute']['osProfile']['computerName']
|
||||
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
|
||||
index c4a8e08d..f8433690 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure.py
|
||||
@@ -1899,6 +1899,56 @@ scbus-1 on xpt0 bus 0
|
||||
dsrc.get_data()
|
||||
self.assertTrue(dsrc.metadata["disable_password"])
|
||||
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_userdata_from_imds(self, m_get_metadata_from_imds):
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+ userdata = "userdataImds"
|
||||
+ imds_data = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data["compute"]["osProfile"] = dict(
|
||||
+ adminUsername="username1",
|
||||
+ computerName="hostname1",
|
||||
+ disablePasswordAuthentication="true",
|
||||
+ )
|
||||
+ imds_data["compute"]["userData"] = b64e(userdata)
|
||||
+ m_get_metadata_from_imds.return_value = imds_data
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ ret = dsrc.get_data()
|
||||
+ self.assertTrue(ret)
|
||||
+ self.assertEqual(dsrc.userdata_raw, userdata.encode('utf-8'))
|
||||
+
|
||||
+ @mock.patch(MOCKPATH + 'get_metadata_from_imds')
|
||||
+ def test_userdata_from_imds_with_customdata_from_OVF(
|
||||
+ self, m_get_metadata_from_imds):
|
||||
+ userdataOVF = "userdataOVF"
|
||||
+ odata = {
|
||||
+ 'HostName': "myhost", 'UserName': "myuser",
|
||||
+ 'UserData': {'text': b64e(userdataOVF), 'encoding': 'base64'}
|
||||
+ }
|
||||
+ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}}
|
||||
+ data = {
|
||||
+ 'ovfcontent': construct_valid_ovf_env(data=odata),
|
||||
+ 'sys_cfg': sys_cfg
|
||||
+ }
|
||||
+
|
||||
+ userdataImds = "userdataImds"
|
||||
+ imds_data = copy.deepcopy(NETWORK_METADATA)
|
||||
+ imds_data["compute"]["osProfile"] = dict(
|
||||
+ adminUsername="username1",
|
||||
+ computerName="hostname1",
|
||||
+ disablePasswordAuthentication="true",
|
||||
+ )
|
||||
+ imds_data["compute"]["userData"] = b64e(userdataImds)
|
||||
+ m_get_metadata_from_imds.return_value = imds_data
|
||||
+ dsrc = self._get_ds(data)
|
||||
+ ret = dsrc.get_data()
|
||||
+ self.assertTrue(ret)
|
||||
+ self.assertEqual(dsrc.userdata_raw, userdataOVF.encode('utf-8'))
|
||||
+
|
||||
|
||||
class TestAzureBounce(CiTestCase):
|
||||
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,177 @@
|
||||
From 2ece71923a37a5e1107c80f091a1cc620943fbf2 Mon Sep 17 00:00:00 2001
|
||||
From: Anh Vo <anhvo@microsoft.com>
|
||||
Date: Fri, 23 Apr 2021 10:18:05 -0400
|
||||
Subject: [PATCH 4/7] Azure: eject the provisioning iso before reporting ready
|
||||
(#861)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [4/7] 63e379a4406530c0c15c733f8eee35421079508b (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
Due to hyper-v implementations, iso ejection is more efficient if performed
|
||||
from within the guest. The code will attempt to perform a best-effort ejection.
|
||||
Failure during ejection will not prevent reporting ready from happening. If iso
|
||||
ejection is successful, later iso ejection from the platform will be a no-op.
|
||||
In the event the iso ejection from the guest fails, iso ejection will still happen at
|
||||
the platform level.
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 22 +++++++++++++++---
|
||||
cloudinit/sources/helpers/azure.py | 23 ++++++++++++++++---
|
||||
.../test_datasource/test_azure_helper.py | 13 +++++++++--
|
||||
3 files changed, 50 insertions(+), 8 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index 020b7006..39e67c4f 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -332,6 +332,7 @@ class DataSourceAzure(sources.DataSource):
|
||||
dsname = 'Azure'
|
||||
_negotiated = False
|
||||
_metadata_imds = sources.UNSET
|
||||
+ _ci_pkl_version = 1
|
||||
|
||||
def __init__(self, sys_cfg, distro, paths):
|
||||
sources.DataSource.__init__(self, sys_cfg, distro, paths)
|
||||
@@ -346,8 +347,13 @@ class DataSourceAzure(sources.DataSource):
|
||||
# Regenerate network config new_instance boot and every boot
|
||||
self.update_events['network'].add(EventType.BOOT)
|
||||
self._ephemeral_dhcp_ctx = None
|
||||
-
|
||||
self.failed_desired_api_version = False
|
||||
+ self.iso_dev = None
|
||||
+
|
||||
+ def _unpickle(self, ci_pkl_version: int) -> None:
|
||||
+ super()._unpickle(ci_pkl_version)
|
||||
+ if "iso_dev" not in self.__dict__:
|
||||
+ self.iso_dev = None
|
||||
|
||||
def __str__(self):
|
||||
root = sources.DataSource.__str__(self)
|
||||
@@ -459,6 +465,13 @@ class DataSourceAzure(sources.DataSource):
|
||||
'%s was not mountable' % cdev, logger_func=LOG.warning)
|
||||
continue
|
||||
|
||||
+ report_diagnostic_event("Found provisioning metadata in %s" % cdev,
|
||||
+ logger_func=LOG.debug)
|
||||
+
|
||||
+ # save the iso device for ejection before reporting ready
|
||||
+ if cdev.startswith("/dev"):
|
||||
+ self.iso_dev = cdev
|
||||
+
|
||||
perform_reprovision = reprovision or self._should_reprovision(ret)
|
||||
perform_reprovision_after_nic_attach = (
|
||||
reprovision_after_nic_attach or
|
||||
@@ -1226,7 +1239,9 @@ class DataSourceAzure(sources.DataSource):
|
||||
@return: The success status of sending the ready signal.
|
||||
"""
|
||||
try:
|
||||
- get_metadata_from_fabric(None, lease['unknown-245'])
|
||||
+ get_metadata_from_fabric(fallback_lease_file=None,
|
||||
+ dhcp_opts=lease['unknown-245'],
|
||||
+ iso_dev=self.iso_dev)
|
||||
return True
|
||||
except Exception as e:
|
||||
report_diagnostic_event(
|
||||
@@ -1332,7 +1347,8 @@ class DataSourceAzure(sources.DataSource):
|
||||
metadata_func = partial(get_metadata_from_fabric,
|
||||
fallback_lease_file=self.
|
||||
dhclient_lease_file,
|
||||
- pubkey_info=pubkey_info)
|
||||
+ pubkey_info=pubkey_info,
|
||||
+ iso_dev=self.iso_dev)
|
||||
|
||||
LOG.debug("negotiating with fabric via agent command %s",
|
||||
self.ds_cfg['agent_command'])
|
||||
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
|
||||
index 03e7156b..ad476076 100755
|
||||
--- a/cloudinit/sources/helpers/azure.py
|
||||
+++ b/cloudinit/sources/helpers/azure.py
|
||||
@@ -865,7 +865,19 @@ class WALinuxAgentShim:
|
||||
return endpoint_ip_address
|
||||
|
||||
@azure_ds_telemetry_reporter
|
||||
- def register_with_azure_and_fetch_data(self, pubkey_info=None) -> dict:
|
||||
+ def eject_iso(self, iso_dev) -> None:
|
||||
+ try:
|
||||
+ LOG.debug("Ejecting the provisioning iso")
|
||||
+ subp.subp(['eject', iso_dev])
|
||||
+ except Exception as e:
|
||||
+ report_diagnostic_event(
|
||||
+ "Failed ejecting the provisioning iso: %s" % e,
|
||||
+ logger_func=LOG.debug)
|
||||
+
|
||||
+ @azure_ds_telemetry_reporter
|
||||
+ def register_with_azure_and_fetch_data(self,
|
||||
+ pubkey_info=None,
|
||||
+ iso_dev=None) -> dict:
|
||||
"""Gets the VM's GoalState from Azure, uses the GoalState information
|
||||
to report ready/send the ready signal/provisioning complete signal to
|
||||
Azure, and then uses pubkey_info to filter and obtain the user's
|
||||
@@ -891,6 +903,10 @@ class WALinuxAgentShim:
|
||||
ssh_keys = self._get_user_pubkeys(goal_state, pubkey_info)
|
||||
health_reporter = GoalStateHealthReporter(
|
||||
goal_state, self.azure_endpoint_client, self.endpoint)
|
||||
+
|
||||
+ if iso_dev is not None:
|
||||
+ self.eject_iso(iso_dev)
|
||||
+
|
||||
health_reporter.send_ready_signal()
|
||||
return {'public-keys': ssh_keys}
|
||||
|
||||
@@ -1046,11 +1062,12 @@ class WALinuxAgentShim:
|
||||
|
||||
@azure_ds_telemetry_reporter
|
||||
def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None,
|
||||
- pubkey_info=None):
|
||||
+ pubkey_info=None, iso_dev=None):
|
||||
shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file,
|
||||
dhcp_options=dhcp_opts)
|
||||
try:
|
||||
- return shim.register_with_azure_and_fetch_data(pubkey_info=pubkey_info)
|
||||
+ return shim.register_with_azure_and_fetch_data(
|
||||
+ pubkey_info=pubkey_info, iso_dev=iso_dev)
|
||||
finally:
|
||||
shim.clean_up()
|
||||
|
||||
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
|
||||
index 63482c6c..552c7905 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure_helper.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure_helper.py
|
||||
@@ -1009,6 +1009,14 @@ class TestWALinuxAgentShim(CiTestCase):
|
||||
self.GoalState.return_value.container_id = self.test_container_id
|
||||
self.GoalState.return_value.instance_id = self.test_instance_id
|
||||
|
||||
+ def test_eject_iso_is_called(self):
|
||||
+ shim = wa_shim()
|
||||
+ with mock.patch.object(
|
||||
+ shim, 'eject_iso', autospec=True
|
||||
+ ) as m_eject_iso:
|
||||
+ shim.register_with_azure_and_fetch_data(iso_dev="/dev/sr0")
|
||||
+ m_eject_iso.assert_called_once_with("/dev/sr0")
|
||||
+
|
||||
def test_http_client_does_not_use_certificate_for_report_ready(self):
|
||||
shim = wa_shim()
|
||||
shim.register_with_azure_and_fetch_data()
|
||||
@@ -1283,13 +1291,14 @@ class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase):
|
||||
|
||||
def test_calls_shim_register_with_azure_and_fetch_data(self):
|
||||
m_pubkey_info = mock.MagicMock()
|
||||
- azure_helper.get_metadata_from_fabric(pubkey_info=m_pubkey_info)
|
||||
+ azure_helper.get_metadata_from_fabric(
|
||||
+ pubkey_info=m_pubkey_info, iso_dev="/dev/sr0")
|
||||
self.assertEqual(
|
||||
1,
|
||||
self.m_shim.return_value
|
||||
.register_with_azure_and_fetch_data.call_count)
|
||||
self.assertEqual(
|
||||
- mock.call(pubkey_info=m_pubkey_info),
|
||||
+ mock.call(iso_dev="/dev/sr0", pubkey_info=m_pubkey_info),
|
||||
self.m_shim.return_value
|
||||
.register_with_azure_and_fetch_data.call_args)
|
||||
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,90 @@
|
||||
From 3ee42e6e6ca51b3fd0b6461f707d62c89d54e227 Mon Sep 17 00:00:00 2001
|
||||
From: Johnson Shi <Johnson.Shi@microsoft.com>
|
||||
Date: Thu, 25 Mar 2021 07:20:10 -0700
|
||||
Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between
|
||||
retries (#842)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [2/7] 65672cdfe2265f32e6d3c440ba5a8accafdb6ca6 (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
Ensure that the Azure helper's http handler sleeps a fixed duration
|
||||
between retry failure attempts. The http handler will sleep a fixed
|
||||
duration between failed attempts regardless of whether the attempt
|
||||
failed due to (1) request timing out or (2) instant failure (no
|
||||
timeout).
|
||||
|
||||
Due to certain platform issues, the http request to the Azure endpoint
|
||||
may instantly fail without reaching the http timeout duration. Without
|
||||
sleeping a fixed duration in between retry attempts, the http handler
|
||||
will loop through the max retry attempts quickly. This causes the
|
||||
communication between cloud-init and the Azure platform to be less
|
||||
resilient due to the short total duration if there is no sleep in
|
||||
between retries.
|
||||
---
|
||||
cloudinit/sources/helpers/azure.py | 2 ++
|
||||
tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++--
|
||||
2 files changed, 11 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
|
||||
index d3055d08..03e7156b 100755
|
||||
--- a/cloudinit/sources/helpers/azure.py
|
||||
+++ b/cloudinit/sources/helpers/azure.py
|
||||
@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str:
|
||||
|
||||
max_readurl_attempts = 240
|
||||
default_readurl_timeout = 5
|
||||
+ sleep_duration_between_retries = 5
|
||||
periodic_logging_attempts = 12
|
||||
|
||||
if 'timeout' not in kwargs:
|
||||
@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str:
|
||||
'attempt %d with exception: %s' %
|
||||
(url, attempt, e),
|
||||
logger_func=LOG.debug)
|
||||
+ time.sleep(sleep_duration_between_retries)
|
||||
|
||||
raise exc
|
||||
|
||||
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
|
||||
index b8899807..63482c6c 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure_helper.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure_helper.py
|
||||
@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||||
|
||||
max_readurl_attempts = 240
|
||||
default_readurl_timeout = 5
|
||||
+ sleep_duration_between_retries = 5
|
||||
periodic_logging_attempts = 12
|
||||
|
||||
def setUp(self):
|
||||
@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||||
self.m_readurl = patches.enter_context(
|
||||
mock.patch.object(
|
||||
azure_helper.url_helper, 'readurl', mock.MagicMock()))
|
||||
- patches.enter_context(
|
||||
- mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock()))
|
||||
+ self.m_sleep = patches.enter_context(
|
||||
+ mock.patch.object(azure_helper.time, 'sleep', autospec=True))
|
||||
|
||||
def test_http_with_retries(self):
|
||||
self.m_readurl.return_value = 'TestResp'
|
||||
@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase):
|
||||
self.m_readurl.call_count,
|
||||
self.periodic_logging_attempts + 1)
|
||||
|
||||
+ # Ensure that cloud-init did sleep between each failed request
|
||||
+ self.assertEqual(
|
||||
+ self.m_sleep.call_count,
|
||||
+ self.periodic_logging_attempts)
|
||||
+ self.m_sleep.assert_called_with(self.sleep_duration_between_retries)
|
||||
+
|
||||
def test_http_with_retries_long_delay_logs_periodic_failure_msg(self):
|
||||
self.m_readurl.side_effect = \
|
||||
[SentinelException] * self.periodic_logging_attempts + \
|
||||
--
|
||||
2.27.0
|
||||
|
47
SOURCES/ci-Change-netifaces-dependency-to-0.10.4-965.patch
Normal file
47
SOURCES/ci-Change-netifaces-dependency-to-0.10.4-965.patch
Normal file
@ -0,0 +1,47 @@
|
||||
From 18138313e009a08592fe79c5e66d6eba8f027f19 Mon Sep 17 00:00:00 2001
|
||||
From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
Date: Fri, 14 Jan 2022 16:49:57 +0100
|
||||
Subject: [PATCH 2/5] Change netifaces dependency to 0.10.4 (#965)
|
||||
|
||||
RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
RH-MergeRequest: 17: Datasource for VMware
|
||||
RH-Commit: [2/5] 8688e8b955a7ee15cf66de0b2a242c7c418b7630 (eesposit/cloud-init-centos-)
|
||||
RH-Bugzilla: 2040090
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
|
||||
|
||||
commit b9d308b4d61d22bacc05bcae59819755975631f8
|
||||
Author: Andrew Kutz <101085+akutz@users.noreply.github.com>
|
||||
Date: Tue Aug 10 15:10:44 2021 -0500
|
||||
|
||||
Change netifaces dependency to 0.10.4 (#965)
|
||||
|
||||
Change netifaces dependency to 0.10.4
|
||||
|
||||
Currently versions Ubuntu <=20.10 use netifaces 0.10.4 By requiring
|
||||
netifaces 0.10.9, the VMware datasource omitted itself from cloud-init
|
||||
on Ubuntu <=20.10.
|
||||
|
||||
This patch changes the netifaces dependency to 0.10.4. While it is true
|
||||
there are patches to netifaces post 0.10.4 that are desirable, testing
|
||||
against the most common network configuration was performed to verify
|
||||
the VMware datasource will still function with netifaces 0.10.4.
|
||||
|
||||
Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
---
|
||||
requirements.txt | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/requirements.txt b/requirements.txt
|
||||
index 41d01d62..c4adc455 100644
|
||||
--- a/requirements.txt
|
||||
+++ b/requirements.txt
|
||||
@@ -40,4 +40,4 @@ jsonschema
|
||||
# and still participate in instance-data by gathering the network in detail at
|
||||
# runtime and merge that information into the metadata and repersist that to
|
||||
# disk.
|
||||
-netifaces>=0.10.9
|
||||
+netifaces>=0.10.4
|
||||
--
|
||||
2.27.0
|
||||
|
2201
SOURCES/ci-Datasource-for-VMware-953.patch
Normal file
2201
SOURCES/ci-Datasource-for-VMware-953.patch
Normal file
File diff suppressed because it is too large
Load Diff
47
SOURCES/ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
Normal file
47
SOURCES/ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
Normal file
@ -0,0 +1,47 @@
|
||||
From 0aba80bf749458960945acf106833b098c3c5c97 Mon Sep 17 00:00:00 2001
|
||||
From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
Date: Fri, 14 Jan 2022 16:50:44 +0100
|
||||
Subject: [PATCH 4/5] Revert unnecesary lcase in ds-identify (#978)
|
||||
|
||||
RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
RH-MergeRequest: 17: Datasource for VMware
|
||||
RH-Commit: [4/5] 334aae223b966173238a905150cf7bc07829c255 (eesposit/cloud-init-centos-)
|
||||
RH-Bugzilla: 2040090
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
|
||||
|
||||
commit f516a7d37c1654addc02485e681b4358d7e7c0db
|
||||
Author: Andrew Kutz <101085+akutz@users.noreply.github.com>
|
||||
Date: Fri Aug 13 14:30:55 2021 -0500
|
||||
|
||||
Revert unnecesary lcase in ds-identify (#978)
|
||||
|
||||
This patch reverts an unnecessary lcase optimization in the
|
||||
ds-identify script. SystemD documents the values produced by
|
||||
the systemd-detect-virt command are lower case, and the mapping
|
||||
table used by the FreeBSD check is also lower-case.
|
||||
|
||||
The optimization added two new forked processes, needlessly
|
||||
causing overhead.
|
||||
|
||||
Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
---
|
||||
tools/ds-identify | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/tools/ds-identify b/tools/ds-identify
|
||||
index 0e12298f..7b782462 100755
|
||||
--- a/tools/ds-identify
|
||||
+++ b/tools/ds-identify
|
||||
@@ -449,7 +449,7 @@ detect_virt() {
|
||||
read_virt() {
|
||||
cached "$DI_VIRT" && return 0
|
||||
detect_virt
|
||||
- DI_VIRT="$(echo "${_RET}" | tr '[:upper:]' '[:lower:]')"
|
||||
+ DI_VIRT="${_RET}"
|
||||
}
|
||||
|
||||
is_container() {
|
||||
--
|
||||
2.27.0
|
||||
|
97
SOURCES/ci-Update-dscheck_VMware-s-rpctool-check-970.patch
Normal file
97
SOURCES/ci-Update-dscheck_VMware-s-rpctool-check-970.patch
Normal file
@ -0,0 +1,97 @@
|
||||
From f284c2925b7076b81afb9207161f01718ba70951 Mon Sep 17 00:00:00 2001
|
||||
From: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
Date: Fri, 14 Jan 2022 16:50:18 +0100
|
||||
Subject: [PATCH 3/5] Update dscheck_VMware's rpctool check (#970)
|
||||
|
||||
RH-Author: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
RH-MergeRequest: 17: Datasource for VMware
|
||||
RH-Commit: [3/5] 0739bc18b46b8877fb3825d13f7cda57acda2dde (eesposit/cloud-init-centos-)
|
||||
RH-Bugzilla: 2040090
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Eduardo Otubo <otubo@redhat.com>
|
||||
|
||||
commit 7781dec3306e9467f216cfcb36b7e10a8b38547a
|
||||
Author: Shreenidhi Shedi <53473811+sshedi@users.noreply.github.com>
|
||||
Date: Fri Aug 13 00:40:39 2021 +0530
|
||||
|
||||
Update dscheck_VMware's rpctool check (#970)
|
||||
|
||||
This patch updates the dscheck_VMware function's use of "vmware-rpctool".
|
||||
|
||||
When checking to see if a "guestinfo" property is set.
|
||||
Because a successful exit code can occur even if there is an empty
|
||||
string returned, it is possible that the VMware datasource will be
|
||||
loaded as a false-positive. This patch ensures that in addition to
|
||||
validating the exit code, the emitted output is also examined to ensure
|
||||
a non-empty value is returned by rpctool before returning "${DS_FOUND}"
|
||||
from "dscheck_VMware()".
|
||||
|
||||
Signed-off-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
---
|
||||
tools/ds-identify | 15 +++++++++------
|
||||
1 file changed, 9 insertions(+), 6 deletions(-)
|
||||
|
||||
diff --git a/tools/ds-identify b/tools/ds-identify
|
||||
index c01eae3d..0e12298f 100755
|
||||
--- a/tools/ds-identify
|
||||
+++ b/tools/ds-identify
|
||||
@@ -141,6 +141,7 @@ error() {
|
||||
debug 0 "$@"
|
||||
stderr "$@"
|
||||
}
|
||||
+
|
||||
warn() {
|
||||
set -- "WARN:" "$@"
|
||||
debug 0 "$@"
|
||||
@@ -344,7 +345,6 @@ geom_label_status_as() {
|
||||
return $ret
|
||||
}
|
||||
|
||||
-
|
||||
read_fs_info_freebsd() {
|
||||
local oifs="$IFS" line="" delim=","
|
||||
local ret=0 labels="" dev="" label="" ftype="" isodevs=""
|
||||
@@ -404,7 +404,6 @@ cached() {
|
||||
[ -n "$1" ] && _RET="$1" && return || return 1
|
||||
}
|
||||
|
||||
-
|
||||
detect_virt() {
|
||||
local virt="${UNAVAILABLE}" r="" out=""
|
||||
if [ -d /run/systemd ]; then
|
||||
@@ -450,7 +449,7 @@ detect_virt() {
|
||||
read_virt() {
|
||||
cached "$DI_VIRT" && return 0
|
||||
detect_virt
|
||||
- DI_VIRT=${_RET}
|
||||
+ DI_VIRT="$(echo "${_RET}" | tr '[:upper:]' '[:lower:]')"
|
||||
}
|
||||
|
||||
is_container() {
|
||||
@@ -1370,16 +1369,20 @@ vmware_has_rpctool() {
|
||||
command -v vmware-rpctool >/dev/null 2>&1
|
||||
}
|
||||
|
||||
+vmware_rpctool_guestinfo() {
|
||||
+ vmware-rpctool "info-get guestinfo.${1}" 2>/dev/null | grep "[[:alnum:]]"
|
||||
+}
|
||||
+
|
||||
vmware_rpctool_guestinfo_metadata() {
|
||||
- vmware-rpctool "info-get guestinfo.metadata"
|
||||
+ vmware_rpctool_guestinfo "metadata"
|
||||
}
|
||||
|
||||
vmware_rpctool_guestinfo_userdata() {
|
||||
- vmware-rpctool "info-get guestinfo.userdata"
|
||||
+ vmware_rpctool_guestinfo "userdata"
|
||||
}
|
||||
|
||||
vmware_rpctool_guestinfo_vendordata() {
|
||||
- vmware-rpctool "info-get guestinfo.vendordata"
|
||||
+ vmware_rpctool_guestinfo "vendordata"
|
||||
}
|
||||
|
||||
dscheck_VMware() {
|
||||
--
|
||||
2.27.0
|
||||
|
@ -0,0 +1,470 @@
|
||||
From 9ccb738cf078555b68122b1fc745a45fe952c439 Mon Sep 17 00:00:00 2001
|
||||
From: Anh Vo <anhvo@microsoft.com>
|
||||
Date: Tue, 13 Apr 2021 17:39:39 -0400
|
||||
Subject: [PATCH 3/7] azure: Removing ability to invoke walinuxagent (#799)
|
||||
|
||||
RH-Author: Eduardo Otubo <otubo@redhat.com>
|
||||
RH-MergeRequest: 18: Add support for userdata on Azure from IMDS
|
||||
RH-Commit: [3/7] 7431b912e3df7ea384820f45e0230b47ab54643c (otubo/cloud-init-src)
|
||||
RH-Bugzilla: 2042351
|
||||
RH-Acked-by: Miroslav Rezanina <mrezanin@redhat.com>
|
||||
RH-Acked-by: Emanuele Giuseppe Esposito <eesposit@redhat.com>
|
||||
|
||||
Invoking walinuxagent from within cloud-init is no longer
|
||||
supported/necessary
|
||||
---
|
||||
cloudinit/sources/DataSourceAzure.py | 137 ++++--------------
|
||||
doc/rtd/topics/datasources/azure.rst | 62 ++------
|
||||
tests/unittests/test_datasource/test_azure.py | 97 -------------
|
||||
3 files changed, 35 insertions(+), 261 deletions(-)
|
||||
|
||||
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
|
||||
index de1452ce..020b7006 100755
|
||||
--- a/cloudinit/sources/DataSourceAzure.py
|
||||
+++ b/cloudinit/sources/DataSourceAzure.py
|
||||
@@ -381,53 +381,6 @@ class DataSourceAzure(sources.DataSource):
|
||||
util.logexc(LOG, "handling set_hostname failed")
|
||||
return False
|
||||
|
||||
- @azure_ds_telemetry_reporter
|
||||
- def get_metadata_from_agent(self):
|
||||
- temp_hostname = self.metadata.get('local-hostname')
|
||||
- agent_cmd = self.ds_cfg['agent_command']
|
||||
- LOG.debug("Getting metadata via agent. hostname=%s cmd=%s",
|
||||
- temp_hostname, agent_cmd)
|
||||
-
|
||||
- self.bounce_network_with_azure_hostname()
|
||||
-
|
||||
- try:
|
||||
- invoke_agent(agent_cmd)
|
||||
- except subp.ProcessExecutionError:
|
||||
- # claim the datasource even if the command failed
|
||||
- util.logexc(LOG, "agent command '%s' failed.",
|
||||
- self.ds_cfg['agent_command'])
|
||||
-
|
||||
- ddir = self.ds_cfg['data_dir']
|
||||
-
|
||||
- fp_files = []
|
||||
- key_value = None
|
||||
- for pk in self.cfg.get('_pubkeys', []):
|
||||
- if pk.get('value', None):
|
||||
- key_value = pk['value']
|
||||
- LOG.debug("SSH authentication: using value from fabric")
|
||||
- else:
|
||||
- bname = str(pk['fingerprint'] + ".crt")
|
||||
- fp_files += [os.path.join(ddir, bname)]
|
||||
- LOG.debug("SSH authentication: "
|
||||
- "using fingerprint from fabric")
|
||||
-
|
||||
- with events.ReportEventStack(
|
||||
- name="waiting-for-ssh-public-key",
|
||||
- description="wait for agents to retrieve SSH keys",
|
||||
- parent=azure_ds_reporter):
|
||||
- # wait very long for public SSH keys to arrive
|
||||
- # https://bugs.launchpad.net/cloud-init/+bug/1717611
|
||||
- missing = util.log_time(logfunc=LOG.debug,
|
||||
- msg="waiting for SSH public key files",
|
||||
- func=util.wait_for_files,
|
||||
- args=(fp_files, 900))
|
||||
- if len(missing):
|
||||
- LOG.warning("Did not find files, but going on: %s", missing)
|
||||
-
|
||||
- metadata = {}
|
||||
- metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files)
|
||||
- return metadata
|
||||
-
|
||||
def _get_subplatform(self):
|
||||
"""Return the subplatform metadata source details."""
|
||||
if self.seed.startswith('/dev'):
|
||||
@@ -1354,35 +1307,32 @@ class DataSourceAzure(sources.DataSource):
|
||||
On failure, returns False.
|
||||
"""
|
||||
|
||||
- if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN:
|
||||
- self.bounce_network_with_azure_hostname()
|
||||
+ self.bounce_network_with_azure_hostname()
|
||||
|
||||
- pubkey_info = None
|
||||
- try:
|
||||
- raise KeyError(
|
||||
- "Not using public SSH keys from IMDS"
|
||||
- )
|
||||
- # pylint:disable=unreachable
|
||||
- public_keys = self.metadata['imds']['compute']['publicKeys']
|
||||
- LOG.debug(
|
||||
- 'Successfully retrieved %s key(s) from IMDS',
|
||||
- len(public_keys)
|
||||
- if public_keys is not None
|
||||
- else 0
|
||||
- )
|
||||
- except KeyError:
|
||||
- LOG.debug(
|
||||
- 'Unable to retrieve SSH keys from IMDS during '
|
||||
- 'negotiation, falling back to OVF'
|
||||
- )
|
||||
- pubkey_info = self.cfg.get('_pubkeys', None)
|
||||
-
|
||||
- metadata_func = partial(get_metadata_from_fabric,
|
||||
- fallback_lease_file=self.
|
||||
- dhclient_lease_file,
|
||||
- pubkey_info=pubkey_info)
|
||||
- else:
|
||||
- metadata_func = self.get_metadata_from_agent
|
||||
+ pubkey_info = None
|
||||
+ try:
|
||||
+ raise KeyError(
|
||||
+ "Not using public SSH keys from IMDS"
|
||||
+ )
|
||||
+ # pylint:disable=unreachable
|
||||
+ public_keys = self.metadata['imds']['compute']['publicKeys']
|
||||
+ LOG.debug(
|
||||
+ 'Successfully retrieved %s key(s) from IMDS',
|
||||
+ len(public_keys)
|
||||
+ if public_keys is not None
|
||||
+ else 0
|
||||
+ )
|
||||
+ except KeyError:
|
||||
+ LOG.debug(
|
||||
+ 'Unable to retrieve SSH keys from IMDS during '
|
||||
+ 'negotiation, falling back to OVF'
|
||||
+ )
|
||||
+ pubkey_info = self.cfg.get('_pubkeys', None)
|
||||
+
|
||||
+ metadata_func = partial(get_metadata_from_fabric,
|
||||
+ fallback_lease_file=self.
|
||||
+ dhclient_lease_file,
|
||||
+ pubkey_info=pubkey_info)
|
||||
|
||||
LOG.debug("negotiating with fabric via agent command %s",
|
||||
self.ds_cfg['agent_command'])
|
||||
@@ -1617,33 +1567,6 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname):
|
||||
return True
|
||||
|
||||
|
||||
-@azure_ds_telemetry_reporter
|
||||
-def crtfile_to_pubkey(fname, data=None):
|
||||
- pipeline = ('openssl x509 -noout -pubkey < "$0" |'
|
||||
- 'ssh-keygen -i -m PKCS8 -f /dev/stdin')
|
||||
- (out, _err) = subp.subp(['sh', '-c', pipeline, fname],
|
||||
- capture=True, data=data)
|
||||
- return out.rstrip()
|
||||
-
|
||||
-
|
||||
-@azure_ds_telemetry_reporter
|
||||
-def pubkeys_from_crt_files(flist):
|
||||
- pubkeys = []
|
||||
- errors = []
|
||||
- for fname in flist:
|
||||
- try:
|
||||
- pubkeys.append(crtfile_to_pubkey(fname))
|
||||
- except subp.ProcessExecutionError:
|
||||
- errors.append(fname)
|
||||
-
|
||||
- if errors:
|
||||
- report_diagnostic_event(
|
||||
- "failed to convert the crt files to pubkey: %s" % errors,
|
||||
- logger_func=LOG.warning)
|
||||
-
|
||||
- return pubkeys
|
||||
-
|
||||
-
|
||||
@azure_ds_telemetry_reporter
|
||||
def write_files(datadir, files, dirmode=None):
|
||||
|
||||
@@ -1672,16 +1595,6 @@ def write_files(datadir, files, dirmode=None):
|
||||
util.write_file(filename=fname, content=content, mode=0o600)
|
||||
|
||||
|
||||
-@azure_ds_telemetry_reporter
|
||||
-def invoke_agent(cmd):
|
||||
- # this is a function itself to simplify patching it for test
|
||||
- if cmd:
|
||||
- LOG.debug("invoking agent: %s", cmd)
|
||||
- subp.subp(cmd, shell=(not isinstance(cmd, list)))
|
||||
- else:
|
||||
- LOG.debug("not invoking agent")
|
||||
-
|
||||
-
|
||||
def find_child(node, filter_func):
|
||||
ret = []
|
||||
if not node.hasChildNodes():
|
||||
diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst
|
||||
index e04c3a33..ad9f2236 100644
|
||||
--- a/doc/rtd/topics/datasources/azure.rst
|
||||
+++ b/doc/rtd/topics/datasources/azure.rst
|
||||
@@ -5,28 +5,6 @@ Azure
|
||||
|
||||
This datasource finds metadata and user-data from the Azure cloud platform.
|
||||
|
||||
-walinuxagent
|
||||
-------------
|
||||
-walinuxagent has several functions within images. For cloud-init
|
||||
-specifically, the relevant functionality it performs is to register the
|
||||
-instance with the Azure cloud platform at boot so networking will be
|
||||
-permitted. For more information about the other functionality of
|
||||
-walinuxagent, see `Azure's documentation
|
||||
-<https://github.com/Azure/WALinuxAgent#introduction>`_ for more details.
|
||||
-(Note, however, that only one of walinuxagent's provisioning and cloud-init
|
||||
-should be used to perform instance customisation.)
|
||||
-
|
||||
-If you are configuring walinuxagent yourself, you will want to ensure that you
|
||||
-have `Provisioning.UseCloudInit
|
||||
-<https://github.com/Azure/WALinuxAgent#provisioningusecloudinit>`_ set to
|
||||
-``y``.
|
||||
-
|
||||
-
|
||||
-Builtin Agent
|
||||
--------------
|
||||
-An alternative to using walinuxagent to register to the Azure cloud platform
|
||||
-is to use the ``__builtin__`` agent command. This section contains more
|
||||
-background on what that code path does, and how to enable it.
|
||||
|
||||
The Azure cloud platform provides initial data to an instance via an attached
|
||||
CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some
|
||||
@@ -41,16 +19,6 @@ by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in
|
||||
'dhclient_hook' of cloud-init itself. This sub-command will write the client
|
||||
information in json format to /run/cloud-init/dhclient.hook/<interface>.json.
|
||||
|
||||
-In order for cloud-init to leverage this method to find the endpoint, the
|
||||
-cloud.cfg file must contain:
|
||||
-
|
||||
-.. sourcecode:: yaml
|
||||
-
|
||||
- datasource:
|
||||
- Azure:
|
||||
- set_hostname: False
|
||||
- agent_command: __builtin__
|
||||
-
|
||||
If those files are not available, the fallback is to check the leases file
|
||||
for the endpoint server (again option 245).
|
||||
|
||||
@@ -83,9 +51,6 @@ configuration (in ``/etc/cloud/cloud.cfg`` or ``/etc/cloud/cloud.cfg.d/``).
|
||||
|
||||
The settings that may be configured are:
|
||||
|
||||
- * **agent_command**: Either __builtin__ (default) or a command to run to getcw
|
||||
- metadata. If __builtin__, get metadata from walinuxagent. Otherwise run the
|
||||
- provided command to obtain metadata.
|
||||
* **apply_network_config**: Boolean set to True to use network configuration
|
||||
described by Azure's IMDS endpoint instead of fallback network config of
|
||||
dhcp on eth0. Default is True. For Ubuntu 16.04 or earlier, default is
|
||||
@@ -121,7 +86,6 @@ An example configuration with the default values is provided below:
|
||||
|
||||
datasource:
|
||||
Azure:
|
||||
- agent_command: __builtin__
|
||||
apply_network_config: true
|
||||
data_dir: /var/lib/waagent
|
||||
dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases
|
||||
@@ -144,9 +108,7 @@ child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``)
|
||||
If both ``UserData`` and ``CustomData`` are provided behavior is undefined on
|
||||
which will be selected.
|
||||
|
||||
-In the example below, user-data provided is 'this is my userdata', and the
|
||||
-datasource config provided is ``{"agent_command": ["start", "walinuxagent"]}``.
|
||||
-That agent command will take affect as if it were specified in system config.
|
||||
+In the example below, user-data provided is 'this is my userdata'
|
||||
|
||||
Example:
|
||||
|
||||
@@ -184,20 +146,16 @@ The hostname is provided to the instance in the ovf-env.xml file as
|
||||
Whatever value the instance provides in its dhcp request will resolve in the
|
||||
domain returned in the 'search' request.
|
||||
|
||||
-The interesting issue is that a generic image will already have a hostname
|
||||
-configured. The ubuntu cloud images have 'ubuntu' as the hostname of the
|
||||
-system, and the initial dhcp request on eth0 is not guaranteed to occur after
|
||||
-the datasource code has been run. So, on first boot, that initial value will
|
||||
-be sent in the dhcp request and *that* value will resolve.
|
||||
-
|
||||
-In order to make the ``HostName`` provided in the ovf-env.xml resolve, a
|
||||
-dhcp request must be made with the new value. Walinuxagent (in its current
|
||||
-version) handles this by polling the state of hostname and bouncing ('``ifdown
|
||||
-eth0; ifup eth0``' the network interface if it sees that a change has been
|
||||
-made.
|
||||
+A generic image will already have a hostname configured. The ubuntu
|
||||
+cloud images have 'ubuntu' as the hostname of the system, and the
|
||||
+initial dhcp request on eth0 is not guaranteed to occur after the
|
||||
+datasource code has been run. So, on first boot, that initial value
|
||||
+will be sent in the dhcp request and *that* value will resolve.
|
||||
|
||||
-cloud-init handles this by setting the hostname in the DataSource's 'get_data'
|
||||
-method via '``hostname $HostName``', and then bouncing the interface. This
|
||||
+In order to make the ``HostName`` provided in the ovf-env.xml resolve,
|
||||
+a dhcp request must be made with the new value. cloud-init handles
|
||||
+this by setting the hostname in the DataSource's 'get_data' method via
|
||||
+'``hostname $HostName``', and then bouncing the interface. This
|
||||
behavior can be configured or disabled in the datasource config. See
|
||||
'Configuration' above.
|
||||
|
||||
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
|
||||
index dedebeb1..320fa857 100644
|
||||
--- a/tests/unittests/test_datasource/test_azure.py
|
||||
+++ b/tests/unittests/test_datasource/test_azure.py
|
||||
@@ -638,17 +638,10 @@ scbus-1 on xpt0 bus 0
|
||||
def dsdevs():
|
||||
return data.get('dsdevs', [])
|
||||
|
||||
- def _invoke_agent(cmd):
|
||||
- data['agent_invoked'] = cmd
|
||||
-
|
||||
def _wait_for_files(flist, _maxwait=None, _naplen=None):
|
||||
data['waited'] = flist
|
||||
return []
|
||||
|
||||
- def _pubkeys_from_crt_files(flist):
|
||||
- data['pubkey_files'] = flist
|
||||
- return ["pubkey_from: %s" % f for f in flist]
|
||||
-
|
||||
if data.get('ovfcontent') is not None:
|
||||
populate_dir(os.path.join(self.paths.seed_dir, "azure"),
|
||||
{'ovf-env.xml': data['ovfcontent']})
|
||||
@@ -675,8 +668,6 @@ scbus-1 on xpt0 bus 0
|
||||
|
||||
self.apply_patches([
|
||||
(dsaz, 'list_possible_azure_ds_devs', dsdevs),
|
||||
- (dsaz, 'invoke_agent', _invoke_agent),
|
||||
- (dsaz, 'pubkeys_from_crt_files', _pubkeys_from_crt_files),
|
||||
(dsaz, 'perform_hostname_bounce', mock.MagicMock()),
|
||||
(dsaz, 'get_hostname', mock.MagicMock()),
|
||||
(dsaz, 'set_hostname', mock.MagicMock()),
|
||||
@@ -765,7 +756,6 @@ scbus-1 on xpt0 bus 0
|
||||
ret = dsrc.get_data()
|
||||
self.m_is_platform_viable.assert_called_with(dsrc.seed_dir)
|
||||
self.assertFalse(ret)
|
||||
- self.assertNotIn('agent_invoked', data)
|
||||
# Assert that for non viable platforms,
|
||||
# there is no communication with the Azure datasource.
|
||||
self.assertEqual(
|
||||
@@ -789,7 +779,6 @@ scbus-1 on xpt0 bus 0
|
||||
ret = dsrc.get_data()
|
||||
self.m_is_platform_viable.assert_called_with(dsrc.seed_dir)
|
||||
self.assertFalse(ret)
|
||||
- self.assertNotIn('agent_invoked', data)
|
||||
self.assertEqual(
|
||||
1,
|
||||
m_report_failure.call_count)
|
||||
@@ -806,7 +795,6 @@ scbus-1 on xpt0 bus 0
|
||||
1,
|
||||
m_crawl_metadata.call_count)
|
||||
self.assertFalse(ret)
|
||||
- self.assertNotIn('agent_invoked', data)
|
||||
|
||||
def test_crawl_metadata_exception_should_report_failure_with_msg(self):
|
||||
data = {}
|
||||
@@ -1086,21 +1074,6 @@ scbus-1 on xpt0 bus 0
|
||||
self.assertTrue(os.path.isdir(self.waagent_d))
|
||||
self.assertEqual(stat.S_IMODE(os.stat(self.waagent_d).st_mode), 0o700)
|
||||
|
||||
- def test_user_cfg_set_agent_command_plain(self):
|
||||
- # set dscfg in via plaintext
|
||||
- # we must have friendly-to-xml formatted plaintext in yaml_cfg
|
||||
- # not all plaintext is expected to work.
|
||||
- yaml_cfg = "{agent_command: my_command}\n"
|
||||
- cfg = yaml.safe_load(yaml_cfg)
|
||||
- odata = {'HostName': "myhost", 'UserName': "myuser",
|
||||
- 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}}
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
|
||||
-
|
||||
- dsrc = self._get_ds(data)
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
- self.assertEqual(data['agent_invoked'], cfg['agent_command'])
|
||||
-
|
||||
@mock.patch('cloudinit.sources.DataSourceAzure.device_driver',
|
||||
return_value=None)
|
||||
def test_network_config_set_from_imds(self, m_driver):
|
||||
@@ -1205,29 +1178,6 @@ scbus-1 on xpt0 bus 0
|
||||
dsrc.get_data()
|
||||
self.assertEqual('eastus2', dsrc.region)
|
||||
|
||||
- def test_user_cfg_set_agent_command(self):
|
||||
- # set dscfg in via base64 encoded yaml
|
||||
- cfg = {'agent_command': "my_command"}
|
||||
- odata = {'HostName': "myhost", 'UserName': "myuser",
|
||||
- 'dscfg': {'text': b64e(yaml.dump(cfg)),
|
||||
- 'encoding': 'base64'}}
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
|
||||
-
|
||||
- dsrc = self._get_ds(data)
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
- self.assertEqual(data['agent_invoked'], cfg['agent_command'])
|
||||
-
|
||||
- def test_sys_cfg_set_agent_command(self):
|
||||
- sys_cfg = {'datasource': {'Azure': {'agent_command': '_COMMAND'}}}
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data={}),
|
||||
- 'sys_cfg': sys_cfg}
|
||||
-
|
||||
- dsrc = self._get_ds(data)
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
- self.assertEqual(data['agent_invoked'], '_COMMAND')
|
||||
-
|
||||
def test_sys_cfg_set_never_destroy_ntfs(self):
|
||||
sys_cfg = {'datasource': {'Azure': {
|
||||
'never_destroy_ntfs': 'user-supplied-value'}}}
|
||||
@@ -1311,51 +1261,6 @@ scbus-1 on xpt0 bus 0
|
||||
self.assertTrue(ret)
|
||||
self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8'))
|
||||
|
||||
- def test_cfg_has_pubkeys_fingerprint(self):
|
||||
- odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}]
|
||||
- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
|
||||
- pubkeys=pubkeys)}
|
||||
-
|
||||
- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
- for mypk in mypklist:
|
||||
- self.assertIn(mypk, dsrc.cfg['_pubkeys'])
|
||||
- self.assertIn('pubkey_from', dsrc.metadata['public-keys'][-1])
|
||||
-
|
||||
- def test_cfg_has_pubkeys_value(self):
|
||||
- # make sure that provided key is used over fingerprint
|
||||
- odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': 'value1'}]
|
||||
- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
|
||||
- pubkeys=pubkeys)}
|
||||
-
|
||||
- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
-
|
||||
- for mypk in mypklist:
|
||||
- self.assertIn(mypk, dsrc.cfg['_pubkeys'])
|
||||
- self.assertIn(mypk['value'], dsrc.metadata['public-keys'])
|
||||
-
|
||||
- def test_cfg_has_no_fingerprint_has_value(self):
|
||||
- # test value is used when fingerprint not provided
|
||||
- odata = {'HostName': "myhost", 'UserName': "myuser"}
|
||||
- mypklist = [{'fingerprint': None, 'path': 'path1', 'value': 'value1'}]
|
||||
- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist]
|
||||
- data = {'ovfcontent': construct_valid_ovf_env(data=odata,
|
||||
- pubkeys=pubkeys)}
|
||||
-
|
||||
- dsrc = self._get_ds(data, agent_command=['not', '__builtin__'])
|
||||
- ret = self._get_and_setup(dsrc)
|
||||
- self.assertTrue(ret)
|
||||
-
|
||||
- for mypk in mypklist:
|
||||
- self.assertIn(mypk['value'], dsrc.metadata['public-keys'])
|
||||
-
|
||||
def test_default_ephemeral_configs_ephemeral_exists(self):
|
||||
# make sure the ephemeral configs are correct if disk present
|
||||
odata = {}
|
||||
@@ -1919,8 +1824,6 @@ class TestAzureBounce(CiTestCase):
|
||||
with_logs = True
|
||||
|
||||
def mock_out_azure_moving_parts(self):
|
||||
- self.patches.enter_context(
|
||||
- mock.patch.object(dsaz, 'invoke_agent'))
|
||||
self.patches.enter_context(
|
||||
mock.patch.object(dsaz.util, 'wait_for_files'))
|
||||
self.patches.enter_context(
|
||||
--
|
||||
2.27.0
|
||||
|
@ -1,6 +1,6 @@
|
||||
Name: cloud-init
|
||||
Version: 21.1
|
||||
Release: 15%{?dist}
|
||||
Release: 17%{?dist}
|
||||
Summary: Cloud instance init scripts
|
||||
License: ASL 2.0 or GPLv3
|
||||
URL: http://launchpad.net/cloud-init
|
||||
@ -34,6 +34,28 @@ Patch13: ci-remove-unnecessary-EOF-string-in-disable-sshd-keygen.patch
|
||||
Patch14: ci-fix-error-on-upgrade-caused-by-new-vendordata2-attri.patch
|
||||
# For bz#2028031 - [RHEL-9] Above 19.2 of cloud-init fails to configure routes when configuring static and default routes to the same destination IP
|
||||
Patch15: ci-cloudinit-net-handle-two-different-routes-for-the-sa.patch
|
||||
# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
|
||||
Patch16: ci-Datasource-for-VMware-953.patch
|
||||
# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
|
||||
Patch17: ci-Change-netifaces-dependency-to-0.10.4-965.patch
|
||||
# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
|
||||
Patch18: ci-Update-dscheck_VMware-s-rpctool-check-970.patch
|
||||
# For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo'
|
||||
Patch19: ci-Revert-unnecesary-lcase-in-ds-identify-978.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch20: ci-Add-flexibility-to-IMDS-api-version-793.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch21: ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch22: ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch23: ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch24: ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch25: ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch
|
||||
# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata
|
||||
Patch26: ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch
|
||||
|
||||
# Source-git patches
|
||||
|
||||
@ -86,6 +108,7 @@ Requires: dhcp-client
|
||||
# https://bugzilla.redhat.com/show_bug.cgi?id=2032524
|
||||
Requires: gdisk
|
||||
Requires: openssl
|
||||
Requires: python3-netifaces
|
||||
|
||||
%{?systemd_requires}
|
||||
|
||||
@ -234,6 +257,26 @@ fi
|
||||
%config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf
|
||||
|
||||
%changelog
|
||||
* Mon Feb 07 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-17
|
||||
- ci-Add-flexibility-to-IMDS-api-version-793.patch [bz#2042351]
|
||||
- ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch [bz#2042351]
|
||||
- ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch [bz#2042351]
|
||||
- ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch [bz#2042351]
|
||||
- ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch [bz#2042351]
|
||||
- ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch [bz#2042351]
|
||||
- ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch [bz#2042351]
|
||||
- Resolves: bz#2042351
|
||||
([RHEL-9] Support for provisioning Azure VM with userdata)
|
||||
|
||||
* Fri Jan 21 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-16
|
||||
- ci-Datasource-for-VMware-953.patch [bz#2040090]
|
||||
- ci-Change-netifaces-dependency-to-0.10.4-965.patch [bz#2040090]
|
||||
- ci-Update-dscheck_VMware-s-rpctool-check-970.patch [bz#2040090]
|
||||
- ci-Revert-unnecesary-lcase-in-ds-identify-978.patch [bz#2040090]
|
||||
- ci-Add-netifaces-package-as-a-Requires-in-cloud-init.sp.patch [bz#2040090]
|
||||
- Resolves: bz#2040090
|
||||
([cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo')
|
||||
|
||||
* Thu Jan 13 2022 Miroslav Rezanina <mrezanin@redhat.com> - 21.1-15
|
||||
- ci-Add-gdisk-and-openssl-as-deps-to-fix-UEFI-Azure-init.patch [bz#2032524]
|
||||
- Resolves: bz#2032524
|
||||
|
Loading…
Reference in New Issue
Block a user