From fb681076fc938ccd005adf60024647a442647fa2 Mon Sep 17 00:00:00 2001 From: Miroslav Rezanina Date: Mon, 7 Feb 2022 05:40:13 -0500 Subject: [PATCH] * Mon Feb 07 2022 Miroslav Rezanina - 21.1-17 - ci-Add-flexibility-to-IMDS-api-version-793.patch [bz#2042351] - ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch [bz#2042351] - ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch [bz#2042351] - ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch [bz#2042351] - ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch [bz#2042351] - ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch [bz#2042351] - ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch [bz#2042351] - Resolves: bz#2042351 ([RHEL-9] Support for provisioning Azure VM with userdata) --- ...-flexibility-to-IMDS-api-version-793.patch | 295 +++++++++++ ...-username-and-hostname-from-IMDS-865.patch | 397 +++++++++++++++ ...metadata-during-nic-attach-for-non-t.patch | 315 ++++++++++++ ...port-for-consuming-userdata-from-IMD.patch | 129 +++++ ...provisioning-iso-before-reporting-re.patch | 177 +++++++ ...ure-Azure-http-handler-sleeps-betwee.patch | 90 ++++ ...g-ability-to-invoke-walinuxagent-799.patch | 470 ++++++++++++++++++ cloud-init.spec | 27 +- 8 files changed, 1899 insertions(+), 1 deletion(-) create mode 100644 ci-Add-flexibility-to-IMDS-api-version-793.patch create mode 100644 ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch create mode 100644 ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch create mode 100644 ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch create mode 100644 ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch create mode 100644 ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch create mode 100644 ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch diff --git a/ci-Add-flexibility-to-IMDS-api-version-793.patch b/ci-Add-flexibility-to-IMDS-api-version-793.patch new file mode 100644 index 0000000..4c3dbc3 --- /dev/null +++ b/ci-Add-flexibility-to-IMDS-api-version-793.patch @@ -0,0 +1,295 @@ +From f844e9c263e59a623ca8c647bd87bf4f91374d54 Mon Sep 17 00:00:00 2001 +From: Thomas Stringer +Date: Wed, 3 Mar 2021 11:07:43 -0500 +Subject: [PATCH 1/7] Add flexibility to IMDS api-version (#793) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [1/7] 99a3db20e3f277a2f12ea21e937e06939434a2ca (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +Add flexibility to IMDS api-version by having both a desired IMDS +api-version and a minimum api-version. The desired api-version will +be used first, and if that fails it will fall back to the minimum +api-version. +--- + cloudinit/sources/DataSourceAzure.py | 113 ++++++++++++++---- + tests/unittests/test_datasource/test_azure.py | 42 ++++++- + 2 files changed, 129 insertions(+), 26 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 553b5a7e..de1452ce 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -78,17 +78,15 @@ AGENT_SEED_DIR = '/var/lib/waagent' + # In the event where the IMDS primary server is not + # available, it takes 1s to fallback to the secondary one + IMDS_TIMEOUT_IN_SECONDS = 2 +-IMDS_URL = "http://169.254.169.254/metadata/" +-IMDS_VER = "2019-06-01" +-IMDS_VER_PARAM = "api-version={}".format(IMDS_VER) ++IMDS_URL = "http://169.254.169.254/metadata" ++IMDS_VER_MIN = "2019-06-01" ++IMDS_VER_WANT = "2020-09-01" + + + class metadata_type(Enum): +- compute = "{}instance?{}".format(IMDS_URL, IMDS_VER_PARAM) +- network = "{}instance/network?{}".format(IMDS_URL, +- IMDS_VER_PARAM) +- reprovisiondata = "{}reprovisiondata?{}".format(IMDS_URL, +- IMDS_VER_PARAM) ++ compute = "{}/instance".format(IMDS_URL) ++ network = "{}/instance/network".format(IMDS_URL) ++ reprovisiondata = "{}/reprovisiondata".format(IMDS_URL) + + + PLATFORM_ENTROPY_SOURCE = "/sys/firmware/acpi/tables/OEM0" +@@ -349,6 +347,8 @@ class DataSourceAzure(sources.DataSource): + self.update_events['network'].add(EventType.BOOT) + self._ephemeral_dhcp_ctx = None + ++ self.failed_desired_api_version = False ++ + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) +@@ -520,8 +520,10 @@ class DataSourceAzure(sources.DataSource): + self._wait_for_all_nics_ready() + ret = self._reprovision() + +- imds_md = get_metadata_from_imds( +- self.fallback_interface, retries=10) ++ imds_md = self.get_imds_data_with_api_fallback( ++ self.fallback_interface, ++ retries=10 ++ ) + (md, userdata_raw, cfg, files) = ret + self.seed = cdev + crawled_data.update({ +@@ -652,6 +654,57 @@ class DataSourceAzure(sources.DataSource): + self.ds_cfg['data_dir'], crawled_data['files'], dirmode=0o700) + return True + ++ @azure_ds_telemetry_reporter ++ def get_imds_data_with_api_fallback( ++ self, ++ fallback_nic, ++ retries, ++ md_type=metadata_type.compute): ++ """ ++ Wrapper for get_metadata_from_imds so that we can have flexibility ++ in which IMDS api-version we use. If a particular instance of IMDS ++ does not have the api version that is desired, we want to make ++ this fault tolerant and fall back to a good known minimum api ++ version. ++ """ ++ ++ if not self.failed_desired_api_version: ++ for _ in range(retries): ++ try: ++ LOG.info( ++ "Attempting IMDS api-version: %s", ++ IMDS_VER_WANT ++ ) ++ return get_metadata_from_imds( ++ fallback_nic=fallback_nic, ++ retries=0, ++ md_type=md_type, ++ api_version=IMDS_VER_WANT ++ ) ++ except UrlError as err: ++ LOG.info( ++ "UrlError with IMDS api-version: %s", ++ IMDS_VER_WANT ++ ) ++ if err.code == 400: ++ log_msg = "Fall back to IMDS api-version: {}".format( ++ IMDS_VER_MIN ++ ) ++ report_diagnostic_event( ++ log_msg, ++ logger_func=LOG.info ++ ) ++ self.failed_desired_api_version = True ++ break ++ ++ LOG.info("Using IMDS api-version: %s", IMDS_VER_MIN) ++ return get_metadata_from_imds( ++ fallback_nic=fallback_nic, ++ retries=retries, ++ md_type=md_type, ++ api_version=IMDS_VER_MIN ++ ) ++ + def device_name_to_device(self, name): + return self.ds_cfg['disk_aliases'].get(name) + +@@ -880,10 +933,11 @@ class DataSourceAzure(sources.DataSource): + # primary nic is being attached first helps here. Otherwise each nic + # could add several seconds of delay. + try: +- imds_md = get_metadata_from_imds( ++ imds_md = self.get_imds_data_with_api_fallback( + ifname, + 5, +- metadata_type.network) ++ metadata_type.network ++ ) + except Exception as e: + LOG.warning( + "Failed to get network metadata using nic %s. Attempt to " +@@ -1017,7 +1071,10 @@ class DataSourceAzure(sources.DataSource): + def _poll_imds(self): + """Poll IMDS for the new provisioning data until we get a valid + response. Then return the returned JSON object.""" +- url = metadata_type.reprovisiondata.value ++ url = "{}?api-version={}".format( ++ metadata_type.reprovisiondata.value, ++ IMDS_VER_MIN ++ ) + headers = {"Metadata": "true"} + nl_sock = None + report_ready = bool(not os.path.isfile(REPORTED_READY_MARKER_FILE)) +@@ -2059,7 +2116,8 @@ def _generate_network_config_from_fallback_config() -> dict: + @azure_ds_telemetry_reporter + def get_metadata_from_imds(fallback_nic, + retries, +- md_type=metadata_type.compute): ++ md_type=metadata_type.compute, ++ api_version=IMDS_VER_MIN): + """Query Azure's instance metadata service, returning a dictionary. + + If network is not up, setup ephemeral dhcp on fallback_nic to talk to the +@@ -2069,13 +2127,16 @@ def get_metadata_from_imds(fallback_nic, + @param fallback_nic: String. The name of the nic which requires active + network in order to query IMDS. + @param retries: The number of retries of the IMDS_URL. ++ @param md_type: Metadata type for IMDS request. ++ @param api_version: IMDS api-version to use in the request. + + @return: A dict of instance metadata containing compute and network + info. + """ + kwargs = {'logfunc': LOG.debug, + 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)', +- 'func': _get_metadata_from_imds, 'args': (retries, md_type,)} ++ 'func': _get_metadata_from_imds, ++ 'args': (retries, md_type, api_version,)} + if net.is_up(fallback_nic): + return util.log_time(**kwargs) + else: +@@ -2091,20 +2152,26 @@ def get_metadata_from_imds(fallback_nic, + + + @azure_ds_telemetry_reporter +-def _get_metadata_from_imds(retries, md_type=metadata_type.compute): +- +- url = md_type.value ++def _get_metadata_from_imds( ++ retries, ++ md_type=metadata_type.compute, ++ api_version=IMDS_VER_MIN): ++ url = "{}?api-version={}".format(md_type.value, api_version) + headers = {"Metadata": "true"} + try: + response = readurl( + url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers, + retries=retries, exception_cb=retry_on_url_exc) + except Exception as e: +- report_diagnostic_event( +- 'Ignoring IMDS instance metadata. ' +- 'Get metadata from IMDS failed: %s' % e, +- logger_func=LOG.warning) +- return {} ++ # pylint:disable=no-member ++ if isinstance(e, UrlError) and e.code == 400: ++ raise ++ else: ++ report_diagnostic_event( ++ 'Ignoring IMDS instance metadata. ' ++ 'Get metadata from IMDS failed: %s' % e, ++ logger_func=LOG.warning) ++ return {} + try: + from json.decoder import JSONDecodeError + json_decode_error = JSONDecodeError +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index f597c723..dedebeb1 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -408,7 +408,9 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + + def setUp(self): + super(TestGetMetadataFromIMDS, self).setUp() +- self.network_md_url = dsaz.IMDS_URL + "instance?api-version=2019-06-01" ++ self.network_md_url = "{}/instance?api-version=2019-06-01".format( ++ dsaz.IMDS_URL ++ ) + + @mock.patch(MOCKPATH + 'readurl') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4', autospec=True) +@@ -518,7 +520,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + """Return empty dict when IMDS network metadata is absent.""" + httpretty.register_uri( + httpretty.GET, +- dsaz.IMDS_URL + 'instance?api-version=2017-12-01', ++ dsaz.IMDS_URL + '/instance?api-version=2017-12-01', + body={}, status=404) + + m_net_is_up.return_value = True # skips dhcp +@@ -1877,6 +1879,40 @@ scbus-1 on xpt0 bus 0 + ssh_keys = dsrc.get_public_ssh_keys() + self.assertEqual(ssh_keys, ['key2']) + ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_imds_api_version_wanted_nonexistent( ++ self, ++ m_get_metadata_from_imds): ++ def get_metadata_from_imds_side_eff(*args, **kwargs): ++ if kwargs['api_version'] == dsaz.IMDS_VER_WANT: ++ raise url_helper.UrlError("No IMDS version", code=400) ++ return NETWORK_METADATA ++ m_get_metadata_from_imds.side_effect = get_metadata_from_imds_side_eff ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ self.assertIsNotNone(dsrc.metadata) ++ self.assertTrue(dsrc.failed_desired_api_version) ++ ++ @mock.patch( ++ MOCKPATH + 'get_metadata_from_imds', return_value=NETWORK_METADATA) ++ def test_imds_api_version_wanted_exists(self, m_get_metadata_from_imds): ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ self.assertIsNotNone(dsrc.metadata) ++ self.assertFalse(dsrc.failed_desired_api_version) ++ + + class TestAzureBounce(CiTestCase): + +@@ -2657,7 +2693,7 @@ class TestPreprovisioningHotAttachNics(CiTestCase): + @mock.patch(MOCKPATH + 'DataSourceAzure.wait_for_link_up') + @mock.patch('cloudinit.sources.helpers.netlink.wait_for_nic_attach_event') + @mock.patch('cloudinit.sources.net.find_fallback_nic') +- @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback') + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') + @mock.patch(MOCKPATH + 'DataSourceAzure._wait_for_nic_detach') + @mock.patch('os.path.isfile') +-- +2.27.0 + diff --git a/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch b/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch new file mode 100644 index 0000000..c30ed09 --- /dev/null +++ b/ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch @@ -0,0 +1,397 @@ +From 68f058e8d20a499f74bc78af8e0c6a90ca57ae20 Mon Sep 17 00:00:00 2001 +From: Thomas Stringer +Date: Mon, 26 Apr 2021 09:41:38 -0400 +Subject: [PATCH 5/7] Azure: Retrieve username and hostname from IMDS (#865) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [5/7] 6a768d31e63e5f00dae0fad2712a7618d62b0879 (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +This change allows us to retrieve the username and hostname from +IMDS instead of having to rely on the mounted OVF. +--- + cloudinit/sources/DataSourceAzure.py | 149 ++++++++++++++---- + tests/unittests/test_datasource/test_azure.py | 87 +++++++++- + 2 files changed, 205 insertions(+), 31 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 39e67c4f..6d7954ee 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -5,6 +5,7 @@ + # This file is part of cloud-init. See LICENSE file for license information. + + import base64 ++from collections import namedtuple + import contextlib + import crypt + from functools import partial +@@ -25,6 +26,7 @@ from cloudinit.net import device_driver + from cloudinit.net.dhcp import EphemeralDHCPv4 + from cloudinit import sources + from cloudinit.sources.helpers import netlink ++from cloudinit import ssh_util + from cloudinit import subp + from cloudinit.url_helper import UrlError, readurl, retry_on_url_exc + from cloudinit import util +@@ -80,7 +82,12 @@ AGENT_SEED_DIR = '/var/lib/waagent' + IMDS_TIMEOUT_IN_SECONDS = 2 + IMDS_URL = "http://169.254.169.254/metadata" + IMDS_VER_MIN = "2019-06-01" +-IMDS_VER_WANT = "2020-09-01" ++IMDS_VER_WANT = "2020-10-01" ++ ++ ++# This holds SSH key data including if the source was ++# from IMDS, as well as the SSH key data itself. ++SSHKeys = namedtuple("SSHKeys", ("keys_from_imds", "ssh_keys")) + + + class metadata_type(Enum): +@@ -391,6 +398,8 @@ class DataSourceAzure(sources.DataSource): + """Return the subplatform metadata source details.""" + if self.seed.startswith('/dev'): + subplatform_type = 'config-disk' ++ elif self.seed.lower() == 'imds': ++ subplatform_type = 'imds' + else: + subplatform_type = 'seed-dir' + return '%s (%s)' % (subplatform_type, self.seed) +@@ -433,9 +442,11 @@ class DataSourceAzure(sources.DataSource): + + found = None + reprovision = False ++ ovf_is_accessible = True + reprovision_after_nic_attach = False + for cdev in candidates: + try: ++ LOG.debug("cdev: %s", cdev) + if cdev == "IMDS": + ret = None + reprovision = True +@@ -462,8 +473,18 @@ class DataSourceAzure(sources.DataSource): + raise sources.InvalidMetaDataException(msg) + except util.MountFailedError: + report_diagnostic_event( +- '%s was not mountable' % cdev, logger_func=LOG.warning) +- continue ++ '%s was not mountable' % cdev, logger_func=LOG.debug) ++ cdev = 'IMDS' ++ ovf_is_accessible = False ++ empty_md = {'local-hostname': ''} ++ empty_cfg = dict( ++ system_info=dict( ++ default_user=dict( ++ name='' ++ ) ++ ) ++ ) ++ ret = (empty_md, '', empty_cfg, {}) + + report_diagnostic_event("Found provisioning metadata in %s" % cdev, + logger_func=LOG.debug) +@@ -490,6 +511,10 @@ class DataSourceAzure(sources.DataSource): + self.fallback_interface, + retries=10 + ) ++ if not imds_md and not ovf_is_accessible: ++ msg = 'No OVF or IMDS available' ++ report_diagnostic_event(msg) ++ raise sources.InvalidMetaDataException(msg) + (md, userdata_raw, cfg, files) = ret + self.seed = cdev + crawled_data.update({ +@@ -498,6 +523,21 @@ class DataSourceAzure(sources.DataSource): + 'metadata': util.mergemanydict( + [md, {'imds': imds_md}]), + 'userdata_raw': userdata_raw}) ++ imds_username = _username_from_imds(imds_md) ++ imds_hostname = _hostname_from_imds(imds_md) ++ imds_disable_password = _disable_password_from_imds(imds_md) ++ if imds_username: ++ LOG.debug('Username retrieved from IMDS: %s', imds_username) ++ cfg['system_info']['default_user']['name'] = imds_username ++ if imds_hostname: ++ LOG.debug('Hostname retrieved from IMDS: %s', imds_hostname) ++ crawled_data['metadata']['local-hostname'] = imds_hostname ++ if imds_disable_password: ++ LOG.debug( ++ 'Disable password retrieved from IMDS: %s', ++ imds_disable_password ++ ) ++ crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501 + found = cdev + + report_diagnostic_event( +@@ -676,6 +716,13 @@ class DataSourceAzure(sources.DataSource): + + @azure_ds_telemetry_reporter + def get_public_ssh_keys(self): ++ """ ++ Retrieve public SSH keys. ++ """ ++ ++ return self._get_public_ssh_keys_and_source().ssh_keys ++ ++ def _get_public_ssh_keys_and_source(self): + """ + Try to get the ssh keys from IMDS first, and if that fails + (i.e. IMDS is unavailable) then fallback to getting the ssh +@@ -685,30 +732,50 @@ class DataSourceAzure(sources.DataSource): + advantage, so this is a strong preference. But we must keep + OVF as a second option for environments that don't have IMDS. + """ ++ + LOG.debug('Retrieving public SSH keys') + ssh_keys = [] ++ keys_from_imds = True ++ LOG.debug('Attempting to get SSH keys from IMDS') + try: +- raise KeyError( +- "Not using public SSH keys from IMDS" +- ) +- # pylint:disable=unreachable + ssh_keys = [ + public_key['keyData'] + for public_key + in self.metadata['imds']['compute']['publicKeys'] + ] +- LOG.debug('Retrieved SSH keys from IMDS') ++ for key in ssh_keys: ++ if not _key_is_openssh_formatted(key=key): ++ keys_from_imds = False ++ break ++ ++ if not keys_from_imds: ++ log_msg = 'Keys not in OpenSSH format, using OVF' ++ else: ++ log_msg = 'Retrieved {} keys from IMDS'.format( ++ len(ssh_keys) ++ if ssh_keys is not None ++ else 0 ++ ) + except KeyError: + log_msg = 'Unable to get keys from IMDS, falling back to OVF' ++ keys_from_imds = False ++ finally: + report_diagnostic_event(log_msg, logger_func=LOG.debug) ++ ++ if not keys_from_imds: ++ LOG.debug('Attempting to get SSH keys from OVF') + try: + ssh_keys = self.metadata['public-keys'] +- LOG.debug('Retrieved keys from OVF') ++ log_msg = 'Retrieved {} keys from OVF'.format(len(ssh_keys)) + except KeyError: + log_msg = 'No keys available from OVF' ++ finally: + report_diagnostic_event(log_msg, logger_func=LOG.debug) + +- return ssh_keys ++ return SSHKeys( ++ keys_from_imds=keys_from_imds, ++ ssh_keys=ssh_keys ++ ) + + def get_config_obj(self): + return self.cfg +@@ -1325,30 +1392,21 @@ class DataSourceAzure(sources.DataSource): + self.bounce_network_with_azure_hostname() + + pubkey_info = None +- try: +- raise KeyError( +- "Not using public SSH keys from IMDS" +- ) +- # pylint:disable=unreachable +- public_keys = self.metadata['imds']['compute']['publicKeys'] +- LOG.debug( +- 'Successfully retrieved %s key(s) from IMDS', +- len(public_keys) +- if public_keys is not None ++ ssh_keys_and_source = self._get_public_ssh_keys_and_source() ++ ++ if not ssh_keys_and_source.keys_from_imds: ++ pubkey_info = self.cfg.get('_pubkeys', None) ++ log_msg = 'Retrieved {} fingerprints from OVF'.format( ++ len(pubkey_info) ++ if pubkey_info is not None + else 0 + ) +- except KeyError: +- LOG.debug( +- 'Unable to retrieve SSH keys from IMDS during ' +- 'negotiation, falling back to OVF' +- ) +- pubkey_info = self.cfg.get('_pubkeys', None) ++ report_diagnostic_event(log_msg, logger_func=LOG.debug) + + metadata_func = partial(get_metadata_from_fabric, + fallback_lease_file=self. + dhclient_lease_file, +- pubkey_info=pubkey_info, +- iso_dev=self.iso_dev) ++ pubkey_info=pubkey_info) + + LOG.debug("negotiating with fabric via agent command %s", + self.ds_cfg['agent_command']) +@@ -1404,6 +1462,41 @@ class DataSourceAzure(sources.DataSource): + return self.metadata.get('imds', {}).get('compute', {}).get('location') + + ++def _username_from_imds(imds_data): ++ try: ++ return imds_data['compute']['osProfile']['adminUsername'] ++ except KeyError: ++ return None ++ ++ ++def _hostname_from_imds(imds_data): ++ try: ++ return imds_data['compute']['osProfile']['computerName'] ++ except KeyError: ++ return None ++ ++ ++def _disable_password_from_imds(imds_data): ++ try: ++ return imds_data['compute']['osProfile']['disablePasswordAuthentication'] == 'true' # noqa: E501 ++ except KeyError: ++ return None ++ ++ ++def _key_is_openssh_formatted(key): ++ """ ++ Validate whether or not the key is OpenSSH-formatted. ++ """ ++ ++ parser = ssh_util.AuthKeyLineParser() ++ try: ++ akl = parser.parse(key) ++ except TypeError: ++ return False ++ ++ return akl.keytype is not None ++ ++ + def _partitions_on_device(devpath, maxnum=16): + # return a list of tuples (ptnum, path) for each part on devpath + for suff in ("-part", "p", ""): +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index 320fa857..d9817d84 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -108,7 +108,7 @@ NETWORK_METADATA = { + "zone": "", + "publicKeys": [ + { +- "keyData": "key1", ++ "keyData": "ssh-rsa key1", + "path": "path1" + } + ] +@@ -1761,8 +1761,29 @@ scbus-1 on xpt0 bus 0 + dsrc.get_data() + dsrc.setup(True) + ssh_keys = dsrc.get_public_ssh_keys() +- # Temporarily alter this test so that SSH public keys +- # from IMDS are *not* going to be in use to fix a regression. ++ self.assertEqual(ssh_keys, ["ssh-rsa key1"]) ++ self.assertEqual(m_parse_certificates.call_count, 0) ++ ++ @mock.patch( ++ 'cloudinit.sources.helpers.azure.OpenSSLManager.parse_certificates') ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_get_public_ssh_keys_with_no_openssh_format( ++ self, ++ m_get_metadata_from_imds, ++ m_parse_certificates): ++ imds_data = copy.deepcopy(NETWORK_METADATA) ++ imds_data['compute']['publicKeys'][0]['keyData'] = 'no-openssh-format' ++ m_get_metadata_from_imds.return_value = imds_data ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ dsrc.setup(True) ++ ssh_keys = dsrc.get_public_ssh_keys() + self.assertEqual(ssh_keys, []) + self.assertEqual(m_parse_certificates.call_count, 0) + +@@ -1818,6 +1839,66 @@ scbus-1 on xpt0 bus 0 + self.assertIsNotNone(dsrc.metadata) + self.assertFalse(dsrc.failed_desired_api_version) + ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_hostname_from_imds(self, m_get_metadata_from_imds): ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) ++ imds_data_with_os_profile["compute"]["osProfile"] = dict( ++ adminUsername="username1", ++ computerName="hostname1", ++ disablePasswordAuthentication="true" ++ ) ++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ self.assertEqual(dsrc.metadata["local-hostname"], "hostname1") ++ ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_username_from_imds(self, m_get_metadata_from_imds): ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) ++ imds_data_with_os_profile["compute"]["osProfile"] = dict( ++ adminUsername="username1", ++ computerName="hostname1", ++ disablePasswordAuthentication="true" ++ ) ++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ self.assertEqual( ++ dsrc.cfg["system_info"]["default_user"]["name"], ++ "username1" ++ ) ++ ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_disable_password_from_imds(self, m_get_metadata_from_imds): ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ imds_data_with_os_profile = copy.deepcopy(NETWORK_METADATA) ++ imds_data_with_os_profile["compute"]["osProfile"] = dict( ++ adminUsername="username1", ++ computerName="hostname1", ++ disablePasswordAuthentication="true" ++ ) ++ m_get_metadata_from_imds.return_value = imds_data_with_os_profile ++ dsrc = self._get_ds(data) ++ dsrc.get_data() ++ self.assertTrue(dsrc.metadata["disable_password"]) ++ + + class TestAzureBounce(CiTestCase): + +-- +2.27.0 + diff --git a/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch b/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch new file mode 100644 index 0000000..2d02d7d --- /dev/null +++ b/ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch @@ -0,0 +1,315 @@ +From 816fe5c2e6d5dcc68f292092b00b2acfbc4c8e88 Mon Sep 17 00:00:00 2001 +From: aswinrajamannar <39812128+aswinrajamannar@users.noreply.github.com> +Date: Mon, 26 Apr 2021 07:28:39 -0700 +Subject: [PATCH 6/7] Azure: Retry net metadata during nic attach for + non-timeout errs (#878) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [6/7] 794cd340644260bb43a7c8582a8067f403b9842d (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +When network interfaces are hot-attached to the VM, attempting to get +network metadata might return 410 (or 500, 503 etc) because the info +is not yet available. In those cases, we retry getting the metadata +before giving up. The only case where we can move on to wait for more +nic attach events is if the call times out despite retries, which +means the interface is not likely a primary interface, and we should +try for more nic attach events. +--- + cloudinit/sources/DataSourceAzure.py | 65 +++++++++++-- + tests/unittests/test_datasource/test_azure.py | 95 ++++++++++++++++--- + 2 files changed, 140 insertions(+), 20 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 6d7954ee..d0be6d84 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -17,6 +17,7 @@ from time import sleep + from xml.dom import minidom + import xml.etree.ElementTree as ET + from enum import Enum ++import requests + + from cloudinit import dmi + from cloudinit import log as logging +@@ -665,7 +666,9 @@ class DataSourceAzure(sources.DataSource): + self, + fallback_nic, + retries, +- md_type=metadata_type.compute): ++ md_type=metadata_type.compute, ++ exc_cb=retry_on_url_exc, ++ infinite=False): + """ + Wrapper for get_metadata_from_imds so that we can have flexibility + in which IMDS api-version we use. If a particular instance of IMDS +@@ -685,7 +688,8 @@ class DataSourceAzure(sources.DataSource): + fallback_nic=fallback_nic, + retries=0, + md_type=md_type, +- api_version=IMDS_VER_WANT ++ api_version=IMDS_VER_WANT, ++ exc_cb=exc_cb + ) + except UrlError as err: + LOG.info( +@@ -708,7 +712,9 @@ class DataSourceAzure(sources.DataSource): + fallback_nic=fallback_nic, + retries=retries, + md_type=md_type, +- api_version=IMDS_VER_MIN ++ api_version=IMDS_VER_MIN, ++ exc_cb=exc_cb, ++ infinite=infinite + ) + + def device_name_to_device(self, name): +@@ -938,6 +944,9 @@ class DataSourceAzure(sources.DataSource): + is_primary = False + expected_nic_count = -1 + imds_md = None ++ metadata_poll_count = 0 ++ metadata_logging_threshold = 1 ++ metadata_timeout_count = 0 + + # For now, only a VM's primary NIC can contact IMDS and WireServer. If + # DHCP fails for a NIC, we have no mechanism to determine if the NIC is +@@ -962,14 +971,48 @@ class DataSourceAzure(sources.DataSource): + % (ifname, e), logger_func=LOG.error) + raise + ++ # Retry polling network metadata for a limited duration only when the ++ # calls fail due to timeout. This is because the platform drops packets ++ # going towards IMDS when it is not a primary nic. If the calls fail ++ # due to other issues like 410, 503 etc, then it means we are primary ++ # but IMDS service is unavailable at the moment. Retry indefinitely in ++ # those cases since we cannot move on without the network metadata. ++ def network_metadata_exc_cb(msg, exc): ++ nonlocal metadata_timeout_count, metadata_poll_count ++ nonlocal metadata_logging_threshold ++ ++ metadata_poll_count = metadata_poll_count + 1 ++ ++ # Log when needed but back off exponentially to avoid exploding ++ # the log file. ++ if metadata_poll_count >= metadata_logging_threshold: ++ metadata_logging_threshold *= 2 ++ report_diagnostic_event( ++ "Ran into exception when attempting to reach %s " ++ "after %d polls." % (msg, metadata_poll_count), ++ logger_func=LOG.error) ++ ++ if isinstance(exc, UrlError): ++ report_diagnostic_event("poll IMDS with %s failed. " ++ "Exception: %s and code: %s" % ++ (msg, exc.cause, exc.code), ++ logger_func=LOG.error) ++ ++ if exc.cause and isinstance(exc.cause, requests.Timeout): ++ metadata_timeout_count = metadata_timeout_count + 1 ++ return (metadata_timeout_count <= 10) ++ return True ++ + # Primary nic detection will be optimized in the future. The fact that + # primary nic is being attached first helps here. Otherwise each nic + # could add several seconds of delay. + try: + imds_md = self.get_imds_data_with_api_fallback( + ifname, +- 5, +- metadata_type.network ++ 0, ++ metadata_type.network, ++ network_metadata_exc_cb, ++ True + ) + except Exception as e: + LOG.warning( +@@ -2139,7 +2182,9 @@ def _generate_network_config_from_fallback_config() -> dict: + def get_metadata_from_imds(fallback_nic, + retries, + md_type=metadata_type.compute, +- api_version=IMDS_VER_MIN): ++ api_version=IMDS_VER_MIN, ++ exc_cb=retry_on_url_exc, ++ infinite=False): + """Query Azure's instance metadata service, returning a dictionary. + + If network is not up, setup ephemeral dhcp on fallback_nic to talk to the +@@ -2158,7 +2203,7 @@ def get_metadata_from_imds(fallback_nic, + kwargs = {'logfunc': LOG.debug, + 'msg': 'Crawl of Azure Instance Metadata Service (IMDS)', + 'func': _get_metadata_from_imds, +- 'args': (retries, md_type, api_version,)} ++ 'args': (retries, exc_cb, md_type, api_version, infinite)} + if net.is_up(fallback_nic): + return util.log_time(**kwargs) + else: +@@ -2176,14 +2221,16 @@ def get_metadata_from_imds(fallback_nic, + @azure_ds_telemetry_reporter + def _get_metadata_from_imds( + retries, ++ exc_cb, + md_type=metadata_type.compute, +- api_version=IMDS_VER_MIN): ++ api_version=IMDS_VER_MIN, ++ infinite=False): + url = "{}?api-version={}".format(md_type.value, api_version) + headers = {"Metadata": "true"} + try: + response = readurl( + url, timeout=IMDS_TIMEOUT_IN_SECONDS, headers=headers, +- retries=retries, exception_cb=retry_on_url_exc) ++ retries=retries, exception_cb=exc_cb, infinite=infinite) + except Exception as e: + # pylint:disable=no-member + if isinstance(e, UrlError) and e.code == 400: +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index d9817d84..c4a8e08d 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -448,7 +448,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + "http://169.254.169.254/metadata/instance?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, +- timeout=mock.ANY) ++ timeout=mock.ANY, infinite=False) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') +@@ -467,7 +467,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + "http://169.254.169.254/metadata/instance/network?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, +- timeout=mock.ANY) ++ timeout=mock.ANY, infinite=False) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4') +@@ -486,7 +486,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + "http://169.254.169.254/metadata/instance?api-version=" + "2019-06-01", exception_cb=mock.ANY, + headers=mock.ANY, retries=mock.ANY, +- timeout=mock.ANY) ++ timeout=mock.ANY, infinite=False) + + @mock.patch(MOCKPATH + 'readurl', autospec=True) + @mock.patch(MOCKPATH + 'EphemeralDHCPv4WithReporting', autospec=True) +@@ -511,7 +511,7 @@ class TestGetMetadataFromIMDS(HttprettyTestCase): + m_readurl.assert_called_with( + self.network_md_url, exception_cb=mock.ANY, + headers={'Metadata': 'true'}, retries=2, +- timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS) ++ timeout=dsaz.IMDS_TIMEOUT_IN_SECONDS, infinite=False) + + @mock.patch('cloudinit.url_helper.time.sleep') + @mock.patch(MOCKPATH + 'net.is_up', autospec=True) +@@ -2694,15 +2694,22 @@ class TestPreprovisioningHotAttachNics(CiTestCase): + + def nic_attach_ret(nl_sock, nics_found): + nonlocal m_attach_call_count +- if m_attach_call_count == 0: +- m_attach_call_count = m_attach_call_count + 1 ++ m_attach_call_count = m_attach_call_count + 1 ++ if m_attach_call_count == 1: + return "eth0" +- return "eth1" ++ elif m_attach_call_count == 2: ++ return "eth1" ++ raise RuntimeError("Must have found primary nic by now.") ++ ++ # Simulate two NICs by adding the same one twice. ++ md = { ++ "interface": [ ++ IMDS_NETWORK_METADATA['interface'][0], ++ IMDS_NETWORK_METADATA['interface'][0] ++ ] ++ } + +- def network_metadata_ret(ifname, retries, type): +- # Simulate two NICs by adding the same one twice. +- md = IMDS_NETWORK_METADATA +- md['interface'].append(md['interface'][0]) ++ def network_metadata_ret(ifname, retries, type, exc_cb, infinite): + if ifname == "eth0": + return md + raise requests.Timeout('Fake connection timeout') +@@ -2724,6 +2731,72 @@ class TestPreprovisioningHotAttachNics(CiTestCase): + self.assertEqual(1, m_imds.call_count) + self.assertEqual(2, m_link_up.call_count) + ++ @mock.patch(MOCKPATH + 'DataSourceAzure.get_imds_data_with_api_fallback') ++ @mock.patch(MOCKPATH + 'EphemeralDHCPv4') ++ def test_check_if_nic_is_primary_retries_on_failures( ++ self, m_dhcpv4, m_imds): ++ """Retry polling for network metadata on all failures except timeout""" ++ dsa = dsaz.DataSourceAzure({}, distro=None, paths=self.paths) ++ lease = { ++ 'interface': 'eth9', 'fixed-address': '192.168.2.9', ++ 'routers': '192.168.2.1', 'subnet-mask': '255.255.255.0', ++ 'unknown-245': '624c3620'} ++ ++ eth0Retries = [] ++ eth1Retries = [] ++ # Simulate two NICs by adding the same one twice. ++ md = { ++ "interface": [ ++ IMDS_NETWORK_METADATA['interface'][0], ++ IMDS_NETWORK_METADATA['interface'][0] ++ ] ++ } ++ ++ def network_metadata_ret(ifname, retries, type, exc_cb, infinite): ++ nonlocal eth0Retries, eth1Retries ++ ++ # Simulate readurl functionality with retries and ++ # exception callbacks so that the callback logic can be ++ # validated. ++ if ifname == "eth0": ++ cause = requests.HTTPError() ++ for _ in range(0, 15): ++ error = url_helper.UrlError(cause=cause, code=410) ++ eth0Retries.append(exc_cb("No goal state.", error)) ++ else: ++ cause = requests.Timeout('Fake connection timeout') ++ for _ in range(0, 10): ++ error = url_helper.UrlError(cause=cause) ++ eth1Retries.append(exc_cb("Connection timeout", error)) ++ # Should stop retrying after 10 retries ++ eth1Retries.append(exc_cb("Connection timeout", error)) ++ raise cause ++ return md ++ ++ m_imds.side_effect = network_metadata_ret ++ ++ dhcp_ctx = mock.MagicMock(lease=lease) ++ dhcp_ctx.obtain_lease.return_value = lease ++ m_dhcpv4.return_value = dhcp_ctx ++ ++ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth0") ++ self.assertEqual(True, is_primary) ++ self.assertEqual(2, expected_nic_count) ++ ++ # All Eth0 errors are non-timeout errors. So we should have been ++ # retrying indefinitely until success. ++ for i in eth0Retries: ++ self.assertTrue(i) ++ ++ is_primary, expected_nic_count = dsa._check_if_nic_is_primary("eth1") ++ self.assertEqual(False, is_primary) ++ ++ # All Eth1 errors are timeout errors. Retry happens for a max of 10 and ++ # then we should have moved on assuming it is not the primary nic. ++ for i in range(0, 10): ++ self.assertTrue(eth1Retries[i]) ++ self.assertFalse(eth1Retries[10]) ++ + @mock.patch('cloudinit.distros.networking.LinuxNetworking.try_set_link_up') + def test_wait_for_link_up_returns_if_already_up( + self, m_is_link_up): +-- +2.27.0 + diff --git a/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch b/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch new file mode 100644 index 0000000..0f17062 --- /dev/null +++ b/ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch @@ -0,0 +1,129 @@ +From 0def71378dc7abf682727c600b696f7313cdcf60 Mon Sep 17 00:00:00 2001 +From: Anh Vo +Date: Tue, 27 Apr 2021 13:40:59 -0400 +Subject: [PATCH 7/7] Azure: adding support for consuming userdata from IMDS + (#884) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [7/7] 1e7ab925162ed9ef2c9b5b9f5c6d5e6ec6e623dd (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito +--- + cloudinit/sources/DataSourceAzure.py | 23 ++++++++- + tests/unittests/test_datasource/test_azure.py | 50 +++++++++++++++++++ + 2 files changed, 72 insertions(+), 1 deletion(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index d0be6d84..a66f023d 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -83,7 +83,7 @@ AGENT_SEED_DIR = '/var/lib/waagent' + IMDS_TIMEOUT_IN_SECONDS = 2 + IMDS_URL = "http://169.254.169.254/metadata" + IMDS_VER_MIN = "2019-06-01" +-IMDS_VER_WANT = "2020-10-01" ++IMDS_VER_WANT = "2021-01-01" + + + # This holds SSH key data including if the source was +@@ -539,6 +539,20 @@ class DataSourceAzure(sources.DataSource): + imds_disable_password + ) + crawled_data['metadata']['disable_password'] = imds_disable_password # noqa: E501 ++ ++ # only use userdata from imds if OVF did not provide custom data ++ # userdata provided by IMDS is always base64 encoded ++ if not userdata_raw: ++ imds_userdata = _userdata_from_imds(imds_md) ++ if imds_userdata: ++ LOG.debug("Retrieved userdata from IMDS") ++ try: ++ crawled_data['userdata_raw'] = base64.b64decode( ++ ''.join(imds_userdata.split())) ++ except Exception: ++ report_diagnostic_event( ++ "Bad userdata in IMDS", ++ logger_func=LOG.warning) + found = cdev + + report_diagnostic_event( +@@ -1512,6 +1526,13 @@ def _username_from_imds(imds_data): + return None + + ++def _userdata_from_imds(imds_data): ++ try: ++ return imds_data['compute']['userData'] ++ except KeyError: ++ return None ++ ++ + def _hostname_from_imds(imds_data): + try: + return imds_data['compute']['osProfile']['computerName'] +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index c4a8e08d..f8433690 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -1899,6 +1899,56 @@ scbus-1 on xpt0 bus 0 + dsrc.get_data() + self.assertTrue(dsrc.metadata["disable_password"]) + ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_userdata_from_imds(self, m_get_metadata_from_imds): ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ odata = {'HostName': "myhost", 'UserName': "myuser"} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ userdata = "userdataImds" ++ imds_data = copy.deepcopy(NETWORK_METADATA) ++ imds_data["compute"]["osProfile"] = dict( ++ adminUsername="username1", ++ computerName="hostname1", ++ disablePasswordAuthentication="true", ++ ) ++ imds_data["compute"]["userData"] = b64e(userdata) ++ m_get_metadata_from_imds.return_value = imds_data ++ dsrc = self._get_ds(data) ++ ret = dsrc.get_data() ++ self.assertTrue(ret) ++ self.assertEqual(dsrc.userdata_raw, userdata.encode('utf-8')) ++ ++ @mock.patch(MOCKPATH + 'get_metadata_from_imds') ++ def test_userdata_from_imds_with_customdata_from_OVF( ++ self, m_get_metadata_from_imds): ++ userdataOVF = "userdataOVF" ++ odata = { ++ 'HostName': "myhost", 'UserName': "myuser", ++ 'UserData': {'text': b64e(userdataOVF), 'encoding': 'base64'} ++ } ++ sys_cfg = {'datasource': {'Azure': {'apply_network_config': True}}} ++ data = { ++ 'ovfcontent': construct_valid_ovf_env(data=odata), ++ 'sys_cfg': sys_cfg ++ } ++ ++ userdataImds = "userdataImds" ++ imds_data = copy.deepcopy(NETWORK_METADATA) ++ imds_data["compute"]["osProfile"] = dict( ++ adminUsername="username1", ++ computerName="hostname1", ++ disablePasswordAuthentication="true", ++ ) ++ imds_data["compute"]["userData"] = b64e(userdataImds) ++ m_get_metadata_from_imds.return_value = imds_data ++ dsrc = self._get_ds(data) ++ ret = dsrc.get_data() ++ self.assertTrue(ret) ++ self.assertEqual(dsrc.userdata_raw, userdataOVF.encode('utf-8')) ++ + + class TestAzureBounce(CiTestCase): + +-- +2.27.0 + diff --git a/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch b/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch new file mode 100644 index 0000000..947a035 --- /dev/null +++ b/ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch @@ -0,0 +1,177 @@ +From 2ece71923a37a5e1107c80f091a1cc620943fbf2 Mon Sep 17 00:00:00 2001 +From: Anh Vo +Date: Fri, 23 Apr 2021 10:18:05 -0400 +Subject: [PATCH 4/7] Azure: eject the provisioning iso before reporting ready + (#861) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [4/7] 63e379a4406530c0c15c733f8eee35421079508b (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +Due to hyper-v implementations, iso ejection is more efficient if performed +from within the guest. The code will attempt to perform a best-effort ejection. +Failure during ejection will not prevent reporting ready from happening. If iso +ejection is successful, later iso ejection from the platform will be a no-op. +In the event the iso ejection from the guest fails, iso ejection will still happen at +the platform level. +--- + cloudinit/sources/DataSourceAzure.py | 22 +++++++++++++++--- + cloudinit/sources/helpers/azure.py | 23 ++++++++++++++++--- + .../test_datasource/test_azure_helper.py | 13 +++++++++-- + 3 files changed, 50 insertions(+), 8 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index 020b7006..39e67c4f 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -332,6 +332,7 @@ class DataSourceAzure(sources.DataSource): + dsname = 'Azure' + _negotiated = False + _metadata_imds = sources.UNSET ++ _ci_pkl_version = 1 + + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) +@@ -346,8 +347,13 @@ class DataSourceAzure(sources.DataSource): + # Regenerate network config new_instance boot and every boot + self.update_events['network'].add(EventType.BOOT) + self._ephemeral_dhcp_ctx = None +- + self.failed_desired_api_version = False ++ self.iso_dev = None ++ ++ def _unpickle(self, ci_pkl_version: int) -> None: ++ super()._unpickle(ci_pkl_version) ++ if "iso_dev" not in self.__dict__: ++ self.iso_dev = None + + def __str__(self): + root = sources.DataSource.__str__(self) +@@ -459,6 +465,13 @@ class DataSourceAzure(sources.DataSource): + '%s was not mountable' % cdev, logger_func=LOG.warning) + continue + ++ report_diagnostic_event("Found provisioning metadata in %s" % cdev, ++ logger_func=LOG.debug) ++ ++ # save the iso device for ejection before reporting ready ++ if cdev.startswith("/dev"): ++ self.iso_dev = cdev ++ + perform_reprovision = reprovision or self._should_reprovision(ret) + perform_reprovision_after_nic_attach = ( + reprovision_after_nic_attach or +@@ -1226,7 +1239,9 @@ class DataSourceAzure(sources.DataSource): + @return: The success status of sending the ready signal. + """ + try: +- get_metadata_from_fabric(None, lease['unknown-245']) ++ get_metadata_from_fabric(fallback_lease_file=None, ++ dhcp_opts=lease['unknown-245'], ++ iso_dev=self.iso_dev) + return True + except Exception as e: + report_diagnostic_event( +@@ -1332,7 +1347,8 @@ class DataSourceAzure(sources.DataSource): + metadata_func = partial(get_metadata_from_fabric, + fallback_lease_file=self. + dhclient_lease_file, +- pubkey_info=pubkey_info) ++ pubkey_info=pubkey_info, ++ iso_dev=self.iso_dev) + + LOG.debug("negotiating with fabric via agent command %s", + self.ds_cfg['agent_command']) +diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py +index 03e7156b..ad476076 100755 +--- a/cloudinit/sources/helpers/azure.py ++++ b/cloudinit/sources/helpers/azure.py +@@ -865,7 +865,19 @@ class WALinuxAgentShim: + return endpoint_ip_address + + @azure_ds_telemetry_reporter +- def register_with_azure_and_fetch_data(self, pubkey_info=None) -> dict: ++ def eject_iso(self, iso_dev) -> None: ++ try: ++ LOG.debug("Ejecting the provisioning iso") ++ subp.subp(['eject', iso_dev]) ++ except Exception as e: ++ report_diagnostic_event( ++ "Failed ejecting the provisioning iso: %s" % e, ++ logger_func=LOG.debug) ++ ++ @azure_ds_telemetry_reporter ++ def register_with_azure_and_fetch_data(self, ++ pubkey_info=None, ++ iso_dev=None) -> dict: + """Gets the VM's GoalState from Azure, uses the GoalState information + to report ready/send the ready signal/provisioning complete signal to + Azure, and then uses pubkey_info to filter and obtain the user's +@@ -891,6 +903,10 @@ class WALinuxAgentShim: + ssh_keys = self._get_user_pubkeys(goal_state, pubkey_info) + health_reporter = GoalStateHealthReporter( + goal_state, self.azure_endpoint_client, self.endpoint) ++ ++ if iso_dev is not None: ++ self.eject_iso(iso_dev) ++ + health_reporter.send_ready_signal() + return {'public-keys': ssh_keys} + +@@ -1046,11 +1062,12 @@ class WALinuxAgentShim: + + @azure_ds_telemetry_reporter + def get_metadata_from_fabric(fallback_lease_file=None, dhcp_opts=None, +- pubkey_info=None): ++ pubkey_info=None, iso_dev=None): + shim = WALinuxAgentShim(fallback_lease_file=fallback_lease_file, + dhcp_options=dhcp_opts) + try: +- return shim.register_with_azure_and_fetch_data(pubkey_info=pubkey_info) ++ return shim.register_with_azure_and_fetch_data( ++ pubkey_info=pubkey_info, iso_dev=iso_dev) + finally: + shim.clean_up() + +diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py +index 63482c6c..552c7905 100644 +--- a/tests/unittests/test_datasource/test_azure_helper.py ++++ b/tests/unittests/test_datasource/test_azure_helper.py +@@ -1009,6 +1009,14 @@ class TestWALinuxAgentShim(CiTestCase): + self.GoalState.return_value.container_id = self.test_container_id + self.GoalState.return_value.instance_id = self.test_instance_id + ++ def test_eject_iso_is_called(self): ++ shim = wa_shim() ++ with mock.patch.object( ++ shim, 'eject_iso', autospec=True ++ ) as m_eject_iso: ++ shim.register_with_azure_and_fetch_data(iso_dev="/dev/sr0") ++ m_eject_iso.assert_called_once_with("/dev/sr0") ++ + def test_http_client_does_not_use_certificate_for_report_ready(self): + shim = wa_shim() + shim.register_with_azure_and_fetch_data() +@@ -1283,13 +1291,14 @@ class TestGetMetadataGoalStateXMLAndReportReadyToFabric(CiTestCase): + + def test_calls_shim_register_with_azure_and_fetch_data(self): + m_pubkey_info = mock.MagicMock() +- azure_helper.get_metadata_from_fabric(pubkey_info=m_pubkey_info) ++ azure_helper.get_metadata_from_fabric( ++ pubkey_info=m_pubkey_info, iso_dev="/dev/sr0") + self.assertEqual( + 1, + self.m_shim.return_value + .register_with_azure_and_fetch_data.call_count) + self.assertEqual( +- mock.call(pubkey_info=m_pubkey_info), ++ mock.call(iso_dev="/dev/sr0", pubkey_info=m_pubkey_info), + self.m_shim.return_value + .register_with_azure_and_fetch_data.call_args) + +-- +2.27.0 + diff --git a/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch b/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch new file mode 100644 index 0000000..33a8acc --- /dev/null +++ b/ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch @@ -0,0 +1,90 @@ +From 3ee42e6e6ca51b3fd0b6461f707d62c89d54e227 Mon Sep 17 00:00:00 2001 +From: Johnson Shi +Date: Thu, 25 Mar 2021 07:20:10 -0700 +Subject: [PATCH 2/7] Azure helper: Ensure Azure http handler sleeps between + retries (#842) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [2/7] 65672cdfe2265f32e6d3c440ba5a8accafdb6ca6 (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +Ensure that the Azure helper's http handler sleeps a fixed duration +between retry failure attempts. The http handler will sleep a fixed +duration between failed attempts regardless of whether the attempt +failed due to (1) request timing out or (2) instant failure (no +timeout). + +Due to certain platform issues, the http request to the Azure endpoint +may instantly fail without reaching the http timeout duration. Without +sleeping a fixed duration in between retry attempts, the http handler +will loop through the max retry attempts quickly. This causes the +communication between cloud-init and the Azure platform to be less +resilient due to the short total duration if there is no sleep in +between retries. +--- + cloudinit/sources/helpers/azure.py | 2 ++ + tests/unittests/test_datasource/test_azure_helper.py | 11 +++++++++-- + 2 files changed, 11 insertions(+), 2 deletions(-) + +diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py +index d3055d08..03e7156b 100755 +--- a/cloudinit/sources/helpers/azure.py ++++ b/cloudinit/sources/helpers/azure.py +@@ -303,6 +303,7 @@ def http_with_retries(url, **kwargs) -> str: + + max_readurl_attempts = 240 + default_readurl_timeout = 5 ++ sleep_duration_between_retries = 5 + periodic_logging_attempts = 12 + + if 'timeout' not in kwargs: +@@ -338,6 +339,7 @@ def http_with_retries(url, **kwargs) -> str: + 'attempt %d with exception: %s' % + (url, attempt, e), + logger_func=LOG.debug) ++ time.sleep(sleep_duration_between_retries) + + raise exc + +diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py +index b8899807..63482c6c 100644 +--- a/tests/unittests/test_datasource/test_azure_helper.py ++++ b/tests/unittests/test_datasource/test_azure_helper.py +@@ -384,6 +384,7 @@ class TestAzureHelperHttpWithRetries(CiTestCase): + + max_readurl_attempts = 240 + default_readurl_timeout = 5 ++ sleep_duration_between_retries = 5 + periodic_logging_attempts = 12 + + def setUp(self): +@@ -394,8 +395,8 @@ class TestAzureHelperHttpWithRetries(CiTestCase): + self.m_readurl = patches.enter_context( + mock.patch.object( + azure_helper.url_helper, 'readurl', mock.MagicMock())) +- patches.enter_context( +- mock.patch.object(azure_helper.time, 'sleep', mock.MagicMock())) ++ self.m_sleep = patches.enter_context( ++ mock.patch.object(azure_helper.time, 'sleep', autospec=True)) + + def test_http_with_retries(self): + self.m_readurl.return_value = 'TestResp' +@@ -438,6 +439,12 @@ class TestAzureHelperHttpWithRetries(CiTestCase): + self.m_readurl.call_count, + self.periodic_logging_attempts + 1) + ++ # Ensure that cloud-init did sleep between each failed request ++ self.assertEqual( ++ self.m_sleep.call_count, ++ self.periodic_logging_attempts) ++ self.m_sleep.assert_called_with(self.sleep_duration_between_retries) ++ + def test_http_with_retries_long_delay_logs_periodic_failure_msg(self): + self.m_readurl.side_effect = \ + [SentinelException] * self.periodic_logging_attempts + \ +-- +2.27.0 + diff --git a/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch b/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch new file mode 100644 index 0000000..17b2187 --- /dev/null +++ b/ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch @@ -0,0 +1,470 @@ +From 9ccb738cf078555b68122b1fc745a45fe952c439 Mon Sep 17 00:00:00 2001 +From: Anh Vo +Date: Tue, 13 Apr 2021 17:39:39 -0400 +Subject: [PATCH 3/7] azure: Removing ability to invoke walinuxagent (#799) + +RH-Author: Eduardo Otubo +RH-MergeRequest: 18: Add support for userdata on Azure from IMDS +RH-Commit: [3/7] 7431b912e3df7ea384820f45e0230b47ab54643c (otubo/cloud-init-src) +RH-Bugzilla: 2042351 +RH-Acked-by: Miroslav Rezanina +RH-Acked-by: Emanuele Giuseppe Esposito + +Invoking walinuxagent from within cloud-init is no longer +supported/necessary +--- + cloudinit/sources/DataSourceAzure.py | 137 ++++-------------- + doc/rtd/topics/datasources/azure.rst | 62 ++------ + tests/unittests/test_datasource/test_azure.py | 97 ------------- + 3 files changed, 35 insertions(+), 261 deletions(-) + +diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py +index de1452ce..020b7006 100755 +--- a/cloudinit/sources/DataSourceAzure.py ++++ b/cloudinit/sources/DataSourceAzure.py +@@ -381,53 +381,6 @@ class DataSourceAzure(sources.DataSource): + util.logexc(LOG, "handling set_hostname failed") + return False + +- @azure_ds_telemetry_reporter +- def get_metadata_from_agent(self): +- temp_hostname = self.metadata.get('local-hostname') +- agent_cmd = self.ds_cfg['agent_command'] +- LOG.debug("Getting metadata via agent. hostname=%s cmd=%s", +- temp_hostname, agent_cmd) +- +- self.bounce_network_with_azure_hostname() +- +- try: +- invoke_agent(agent_cmd) +- except subp.ProcessExecutionError: +- # claim the datasource even if the command failed +- util.logexc(LOG, "agent command '%s' failed.", +- self.ds_cfg['agent_command']) +- +- ddir = self.ds_cfg['data_dir'] +- +- fp_files = [] +- key_value = None +- for pk in self.cfg.get('_pubkeys', []): +- if pk.get('value', None): +- key_value = pk['value'] +- LOG.debug("SSH authentication: using value from fabric") +- else: +- bname = str(pk['fingerprint'] + ".crt") +- fp_files += [os.path.join(ddir, bname)] +- LOG.debug("SSH authentication: " +- "using fingerprint from fabric") +- +- with events.ReportEventStack( +- name="waiting-for-ssh-public-key", +- description="wait for agents to retrieve SSH keys", +- parent=azure_ds_reporter): +- # wait very long for public SSH keys to arrive +- # https://bugs.launchpad.net/cloud-init/+bug/1717611 +- missing = util.log_time(logfunc=LOG.debug, +- msg="waiting for SSH public key files", +- func=util.wait_for_files, +- args=(fp_files, 900)) +- if len(missing): +- LOG.warning("Did not find files, but going on: %s", missing) +- +- metadata = {} +- metadata['public-keys'] = key_value or pubkeys_from_crt_files(fp_files) +- return metadata +- + def _get_subplatform(self): + """Return the subplatform metadata source details.""" + if self.seed.startswith('/dev'): +@@ -1354,35 +1307,32 @@ class DataSourceAzure(sources.DataSource): + On failure, returns False. + """ + +- if self.ds_cfg['agent_command'] == AGENT_START_BUILTIN: +- self.bounce_network_with_azure_hostname() ++ self.bounce_network_with_azure_hostname() + +- pubkey_info = None +- try: +- raise KeyError( +- "Not using public SSH keys from IMDS" +- ) +- # pylint:disable=unreachable +- public_keys = self.metadata['imds']['compute']['publicKeys'] +- LOG.debug( +- 'Successfully retrieved %s key(s) from IMDS', +- len(public_keys) +- if public_keys is not None +- else 0 +- ) +- except KeyError: +- LOG.debug( +- 'Unable to retrieve SSH keys from IMDS during ' +- 'negotiation, falling back to OVF' +- ) +- pubkey_info = self.cfg.get('_pubkeys', None) +- +- metadata_func = partial(get_metadata_from_fabric, +- fallback_lease_file=self. +- dhclient_lease_file, +- pubkey_info=pubkey_info) +- else: +- metadata_func = self.get_metadata_from_agent ++ pubkey_info = None ++ try: ++ raise KeyError( ++ "Not using public SSH keys from IMDS" ++ ) ++ # pylint:disable=unreachable ++ public_keys = self.metadata['imds']['compute']['publicKeys'] ++ LOG.debug( ++ 'Successfully retrieved %s key(s) from IMDS', ++ len(public_keys) ++ if public_keys is not None ++ else 0 ++ ) ++ except KeyError: ++ LOG.debug( ++ 'Unable to retrieve SSH keys from IMDS during ' ++ 'negotiation, falling back to OVF' ++ ) ++ pubkey_info = self.cfg.get('_pubkeys', None) ++ ++ metadata_func = partial(get_metadata_from_fabric, ++ fallback_lease_file=self. ++ dhclient_lease_file, ++ pubkey_info=pubkey_info) + + LOG.debug("negotiating with fabric via agent command %s", + self.ds_cfg['agent_command']) +@@ -1617,33 +1567,6 @@ def perform_hostname_bounce(hostname, cfg, prev_hostname): + return True + + +-@azure_ds_telemetry_reporter +-def crtfile_to_pubkey(fname, data=None): +- pipeline = ('openssl x509 -noout -pubkey < "$0" |' +- 'ssh-keygen -i -m PKCS8 -f /dev/stdin') +- (out, _err) = subp.subp(['sh', '-c', pipeline, fname], +- capture=True, data=data) +- return out.rstrip() +- +- +-@azure_ds_telemetry_reporter +-def pubkeys_from_crt_files(flist): +- pubkeys = [] +- errors = [] +- for fname in flist: +- try: +- pubkeys.append(crtfile_to_pubkey(fname)) +- except subp.ProcessExecutionError: +- errors.append(fname) +- +- if errors: +- report_diagnostic_event( +- "failed to convert the crt files to pubkey: %s" % errors, +- logger_func=LOG.warning) +- +- return pubkeys +- +- + @azure_ds_telemetry_reporter + def write_files(datadir, files, dirmode=None): + +@@ -1672,16 +1595,6 @@ def write_files(datadir, files, dirmode=None): + util.write_file(filename=fname, content=content, mode=0o600) + + +-@azure_ds_telemetry_reporter +-def invoke_agent(cmd): +- # this is a function itself to simplify patching it for test +- if cmd: +- LOG.debug("invoking agent: %s", cmd) +- subp.subp(cmd, shell=(not isinstance(cmd, list))) +- else: +- LOG.debug("not invoking agent") +- +- + def find_child(node, filter_func): + ret = [] + if not node.hasChildNodes(): +diff --git a/doc/rtd/topics/datasources/azure.rst b/doc/rtd/topics/datasources/azure.rst +index e04c3a33..ad9f2236 100644 +--- a/doc/rtd/topics/datasources/azure.rst ++++ b/doc/rtd/topics/datasources/azure.rst +@@ -5,28 +5,6 @@ Azure + + This datasource finds metadata and user-data from the Azure cloud platform. + +-walinuxagent +------------- +-walinuxagent has several functions within images. For cloud-init +-specifically, the relevant functionality it performs is to register the +-instance with the Azure cloud platform at boot so networking will be +-permitted. For more information about the other functionality of +-walinuxagent, see `Azure's documentation +-`_ for more details. +-(Note, however, that only one of walinuxagent's provisioning and cloud-init +-should be used to perform instance customisation.) +- +-If you are configuring walinuxagent yourself, you will want to ensure that you +-have `Provisioning.UseCloudInit +-`_ set to +-``y``. +- +- +-Builtin Agent +-------------- +-An alternative to using walinuxagent to register to the Azure cloud platform +-is to use the ``__builtin__`` agent command. This section contains more +-background on what that code path does, and how to enable it. + + The Azure cloud platform provides initial data to an instance via an attached + CD formatted in UDF. That CD contains a 'ovf-env.xml' file that provides some +@@ -41,16 +19,6 @@ by calling a script in /etc/dhcp/dhclient-exit-hooks or a file in + 'dhclient_hook' of cloud-init itself. This sub-command will write the client + information in json format to /run/cloud-init/dhclient.hook/.json. + +-In order for cloud-init to leverage this method to find the endpoint, the +-cloud.cfg file must contain: +- +-.. sourcecode:: yaml +- +- datasource: +- Azure: +- set_hostname: False +- agent_command: __builtin__ +- + If those files are not available, the fallback is to check the leases file + for the endpoint server (again option 245). + +@@ -83,9 +51,6 @@ configuration (in ``/etc/cloud/cloud.cfg`` or ``/etc/cloud/cloud.cfg.d/``). + + The settings that may be configured are: + +- * **agent_command**: Either __builtin__ (default) or a command to run to getcw +- metadata. If __builtin__, get metadata from walinuxagent. Otherwise run the +- provided command to obtain metadata. + * **apply_network_config**: Boolean set to True to use network configuration + described by Azure's IMDS endpoint instead of fallback network config of + dhcp on eth0. Default is True. For Ubuntu 16.04 or earlier, default is +@@ -121,7 +86,6 @@ An example configuration with the default values is provided below: + + datasource: + Azure: +- agent_command: __builtin__ + apply_network_config: true + data_dir: /var/lib/waagent + dhclient_lease_file: /var/lib/dhcp/dhclient.eth0.leases +@@ -144,9 +108,7 @@ child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``) + If both ``UserData`` and ``CustomData`` are provided behavior is undefined on + which will be selected. + +-In the example below, user-data provided is 'this is my userdata', and the +-datasource config provided is ``{"agent_command": ["start", "walinuxagent"]}``. +-That agent command will take affect as if it were specified in system config. ++In the example below, user-data provided is 'this is my userdata' + + Example: + +@@ -184,20 +146,16 @@ The hostname is provided to the instance in the ovf-env.xml file as + Whatever value the instance provides in its dhcp request will resolve in the + domain returned in the 'search' request. + +-The interesting issue is that a generic image will already have a hostname +-configured. The ubuntu cloud images have 'ubuntu' as the hostname of the +-system, and the initial dhcp request on eth0 is not guaranteed to occur after +-the datasource code has been run. So, on first boot, that initial value will +-be sent in the dhcp request and *that* value will resolve. +- +-In order to make the ``HostName`` provided in the ovf-env.xml resolve, a +-dhcp request must be made with the new value. Walinuxagent (in its current +-version) handles this by polling the state of hostname and bouncing ('``ifdown +-eth0; ifup eth0``' the network interface if it sees that a change has been +-made. ++A generic image will already have a hostname configured. The ubuntu ++cloud images have 'ubuntu' as the hostname of the system, and the ++initial dhcp request on eth0 is not guaranteed to occur after the ++datasource code has been run. So, on first boot, that initial value ++will be sent in the dhcp request and *that* value will resolve. + +-cloud-init handles this by setting the hostname in the DataSource's 'get_data' +-method via '``hostname $HostName``', and then bouncing the interface. This ++In order to make the ``HostName`` provided in the ovf-env.xml resolve, ++a dhcp request must be made with the new value. cloud-init handles ++this by setting the hostname in the DataSource's 'get_data' method via ++'``hostname $HostName``', and then bouncing the interface. This + behavior can be configured or disabled in the datasource config. See + 'Configuration' above. + +diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py +index dedebeb1..320fa857 100644 +--- a/tests/unittests/test_datasource/test_azure.py ++++ b/tests/unittests/test_datasource/test_azure.py +@@ -638,17 +638,10 @@ scbus-1 on xpt0 bus 0 + def dsdevs(): + return data.get('dsdevs', []) + +- def _invoke_agent(cmd): +- data['agent_invoked'] = cmd +- + def _wait_for_files(flist, _maxwait=None, _naplen=None): + data['waited'] = flist + return [] + +- def _pubkeys_from_crt_files(flist): +- data['pubkey_files'] = flist +- return ["pubkey_from: %s" % f for f in flist] +- + if data.get('ovfcontent') is not None: + populate_dir(os.path.join(self.paths.seed_dir, "azure"), + {'ovf-env.xml': data['ovfcontent']}) +@@ -675,8 +668,6 @@ scbus-1 on xpt0 bus 0 + + self.apply_patches([ + (dsaz, 'list_possible_azure_ds_devs', dsdevs), +- (dsaz, 'invoke_agent', _invoke_agent), +- (dsaz, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), + (dsaz, 'perform_hostname_bounce', mock.MagicMock()), + (dsaz, 'get_hostname', mock.MagicMock()), + (dsaz, 'set_hostname', mock.MagicMock()), +@@ -765,7 +756,6 @@ scbus-1 on xpt0 bus 0 + ret = dsrc.get_data() + self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + self.assertFalse(ret) +- self.assertNotIn('agent_invoked', data) + # Assert that for non viable platforms, + # there is no communication with the Azure datasource. + self.assertEqual( +@@ -789,7 +779,6 @@ scbus-1 on xpt0 bus 0 + ret = dsrc.get_data() + self.m_is_platform_viable.assert_called_with(dsrc.seed_dir) + self.assertFalse(ret) +- self.assertNotIn('agent_invoked', data) + self.assertEqual( + 1, + m_report_failure.call_count) +@@ -806,7 +795,6 @@ scbus-1 on xpt0 bus 0 + 1, + m_crawl_metadata.call_count) + self.assertFalse(ret) +- self.assertNotIn('agent_invoked', data) + + def test_crawl_metadata_exception_should_report_failure_with_msg(self): + data = {} +@@ -1086,21 +1074,6 @@ scbus-1 on xpt0 bus 0 + self.assertTrue(os.path.isdir(self.waagent_d)) + self.assertEqual(stat.S_IMODE(os.stat(self.waagent_d).st_mode), 0o700) + +- def test_user_cfg_set_agent_command_plain(self): +- # set dscfg in via plaintext +- # we must have friendly-to-xml formatted plaintext in yaml_cfg +- # not all plaintext is expected to work. +- yaml_cfg = "{agent_command: my_command}\n" +- cfg = yaml.safe_load(yaml_cfg) +- odata = {'HostName': "myhost", 'UserName': "myuser", +- 'dscfg': {'text': yaml_cfg, 'encoding': 'plain'}} +- data = {'ovfcontent': construct_valid_ovf_env(data=odata)} +- +- dsrc = self._get_ds(data) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- self.assertEqual(data['agent_invoked'], cfg['agent_command']) +- + @mock.patch('cloudinit.sources.DataSourceAzure.device_driver', + return_value=None) + def test_network_config_set_from_imds(self, m_driver): +@@ -1205,29 +1178,6 @@ scbus-1 on xpt0 bus 0 + dsrc.get_data() + self.assertEqual('eastus2', dsrc.region) + +- def test_user_cfg_set_agent_command(self): +- # set dscfg in via base64 encoded yaml +- cfg = {'agent_command': "my_command"} +- odata = {'HostName': "myhost", 'UserName': "myuser", +- 'dscfg': {'text': b64e(yaml.dump(cfg)), +- 'encoding': 'base64'}} +- data = {'ovfcontent': construct_valid_ovf_env(data=odata)} +- +- dsrc = self._get_ds(data) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- self.assertEqual(data['agent_invoked'], cfg['agent_command']) +- +- def test_sys_cfg_set_agent_command(self): +- sys_cfg = {'datasource': {'Azure': {'agent_command': '_COMMAND'}}} +- data = {'ovfcontent': construct_valid_ovf_env(data={}), +- 'sys_cfg': sys_cfg} +- +- dsrc = self._get_ds(data) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- self.assertEqual(data['agent_invoked'], '_COMMAND') +- + def test_sys_cfg_set_never_destroy_ntfs(self): + sys_cfg = {'datasource': {'Azure': { + 'never_destroy_ntfs': 'user-supplied-value'}}} +@@ -1311,51 +1261,6 @@ scbus-1 on xpt0 bus 0 + self.assertTrue(ret) + self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8')) + +- def test_cfg_has_pubkeys_fingerprint(self): +- odata = {'HostName': "myhost", 'UserName': "myuser"} +- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': ''}] +- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist] +- data = {'ovfcontent': construct_valid_ovf_env(data=odata, +- pubkeys=pubkeys)} +- +- dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- for mypk in mypklist: +- self.assertIn(mypk, dsrc.cfg['_pubkeys']) +- self.assertIn('pubkey_from', dsrc.metadata['public-keys'][-1]) +- +- def test_cfg_has_pubkeys_value(self): +- # make sure that provided key is used over fingerprint +- odata = {'HostName': "myhost", 'UserName': "myuser"} +- mypklist = [{'fingerprint': 'fp1', 'path': 'path1', 'value': 'value1'}] +- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist] +- data = {'ovfcontent': construct_valid_ovf_env(data=odata, +- pubkeys=pubkeys)} +- +- dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- +- for mypk in mypklist: +- self.assertIn(mypk, dsrc.cfg['_pubkeys']) +- self.assertIn(mypk['value'], dsrc.metadata['public-keys']) +- +- def test_cfg_has_no_fingerprint_has_value(self): +- # test value is used when fingerprint not provided +- odata = {'HostName': "myhost", 'UserName': "myuser"} +- mypklist = [{'fingerprint': None, 'path': 'path1', 'value': 'value1'}] +- pubkeys = [(x['fingerprint'], x['path'], x['value']) for x in mypklist] +- data = {'ovfcontent': construct_valid_ovf_env(data=odata, +- pubkeys=pubkeys)} +- +- dsrc = self._get_ds(data, agent_command=['not', '__builtin__']) +- ret = self._get_and_setup(dsrc) +- self.assertTrue(ret) +- +- for mypk in mypklist: +- self.assertIn(mypk['value'], dsrc.metadata['public-keys']) +- + def test_default_ephemeral_configs_ephemeral_exists(self): + # make sure the ephemeral configs are correct if disk present + odata = {} +@@ -1919,8 +1824,6 @@ class TestAzureBounce(CiTestCase): + with_logs = True + + def mock_out_azure_moving_parts(self): +- self.patches.enter_context( +- mock.patch.object(dsaz, 'invoke_agent')) + self.patches.enter_context( + mock.patch.object(dsaz.util, 'wait_for_files')) + self.patches.enter_context( +-- +2.27.0 + diff --git a/cloud-init.spec b/cloud-init.spec index c39134b..c16a07c 100644 --- a/cloud-init.spec +++ b/cloud-init.spec @@ -1,6 +1,6 @@ Name: cloud-init Version: 21.1 -Release: 16%{?dist} +Release: 17%{?dist} Summary: Cloud instance init scripts License: ASL 2.0 or GPLv3 URL: http://launchpad.net/cloud-init @@ -42,6 +42,20 @@ Patch17: ci-Change-netifaces-dependency-to-0.10.4-965.patch Patch18: ci-Update-dscheck_VMware-s-rpctool-check-970.patch # For bz#2040090 - [cloud-init][RHEL9] Support for cloud-init datasource 'cloud-init-vmware-guestinfo' Patch19: ci-Revert-unnecesary-lcase-in-ds-identify-978.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch20: ci-Add-flexibility-to-IMDS-api-version-793.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch21: ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch22: ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch23: ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch24: ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch25: ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch +# For bz#2042351 - [RHEL-9] Support for provisioning Azure VM with userdata +Patch26: ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch # Source-git patches @@ -243,6 +257,17 @@ fi %config(noreplace) %{_sysconfdir}/rsyslog.d/21-cloudinit.conf %changelog +* Mon Feb 07 2022 Miroslav Rezanina - 21.1-17 +- ci-Add-flexibility-to-IMDS-api-version-793.patch [bz#2042351] +- ci-Azure-helper-Ensure-Azure-http-handler-sleeps-betwee.patch [bz#2042351] +- ci-azure-Removing-ability-to-invoke-walinuxagent-799.patch [bz#2042351] +- ci-Azure-eject-the-provisioning-iso-before-reporting-re.patch [bz#2042351] +- ci-Azure-Retrieve-username-and-hostname-from-IMDS-865.patch [bz#2042351] +- ci-Azure-Retry-net-metadata-during-nic-attach-for-non-t.patch [bz#2042351] +- ci-Azure-adding-support-for-consuming-userdata-from-IMD.patch [bz#2042351] +- Resolves: bz#2042351 + ([RHEL-9] Support for provisioning Azure VM with userdata) + * Fri Jan 21 2022 Miroslav Rezanina - 21.1-16 - ci-Datasource-for-VMware-953.patch [bz#2040090] - ci-Change-netifaces-dependency-to-0.10.4-965.patch [bz#2040090]