diff --git a/services/plugins/vmbackup/stateMachine.c b/services/plugins/vmbackup/stateMachine.c index ebeaf42b6..14d08a77f 100644 --- a/services/plugins/vmbackup/stateMachine.c +++ b/services/plugins/vmbackup/stateMachine.c @@ -453,12 +453,13 @@ VmBackupDoAbort(void) g_static_mutex_unlock(&gBackupState->opLock); #ifdef __linux__ - /* Thaw the guest if already quiesced */ + /* If quiescing has been completed, then undo it. */ if (gBackupState->machineState == VMBACKUP_MSTATE_SYNC_FREEZE) { - g_debug("Guest already quiesced, thawing for abort\n"); - if (!gBackupState->provider->snapshotDone(gBackupState, + g_debug("Aborting with file system already quiesced, undo quiescing " + "operation.\n"); + if (!gBackupState->provider->undo(gBackupState, gBackupState->provider->clientData)) { - g_debug("Thaw during abort failed\n"); + g_debug("Quiescing undo failed.\n"); eventMsg = "Quiesce could not be aborted."; } } diff --git a/services/plugins/vmbackup/syncDriverOps.c b/services/plugins/vmbackup/syncDriverOps.c index 9f5844439..1dd98e323 100644 --- a/services/plugins/vmbackup/syncDriverOps.c +++ b/services/plugins/vmbackup/syncDriverOps.c @@ -35,16 +35,53 @@ #include #endif +/* + * Define an enumeration type VmBackupOpType and a corresponding array + * VmBackupOpName whose entries provide the printable names of the + * enumeration ids in VmBackupOpType. + * + * VmBackupOpType and VmBackupOpName are each defined as an invocation + * of a macro VMBACKUP_OPLIST. VMBACKUP_OPLIST specifies a list of + * enumeration ids using a macro VMBACKUP_OP that must be defined before + * invoking VMBACKUP_OPLIST. VMBACKUP_OP takes a single argument, which + * should be an enumeration id, and is defined to generate from the id + * either the id itself or a string to be used as its printable name. The + * result is that an invocation of VMBACKUP_OPLIST generates either the + * list of enumeration ids or the list of their printable names. + */ +#define VMBACKUP_OPLIST \ + VMBACKUP_OP(OP_FREEZE), \ + VMBACKUP_OP(OP_THAW), \ + VMBACKUP_OP(OP_UNDO), + +#define VMBACKUP_OPID(id) id +#define VMBACKUP_OPNAME(id) #id + +#undef VMBACKUP_OP +#define VMBACKUP_OP(id) VMBACKUP_OPID(id) + +typedef enum { + VMBACKUP_OPLIST +} VmBackupOpType; + +#undef VMBACKUP_OP +#define VMBACKUP_OP(id) VMBACKUP_OPNAME(id) + +static const char *VmBackupOpName[] = { + VMBACKUP_OPLIST +}; + +#undef VMBACKUP_OP + typedef struct VmBackupDriverOp { VmBackupOp callbacks; const char *volumes; - Bool freeze; + VmBackupOpType opType; Bool canceled; SyncDriverHandle *syncHandle; SyncManifest *manifest; } VmBackupDriverOp; - /* *----------------------------------------------------------------------------- * @@ -97,7 +134,7 @@ VmBackupDriverOpQuery(VmBackupOp *_op) // IN VmBackupDriverOp *op = (VmBackupDriverOp *) _op; VmBackupOpStatus ret; - if (op->freeze) { + if (op->opType == OP_FREEZE) { SyncDriverStatus st = SyncDriver_QueryStatus(*op->syncHandle, 0); g_debug("SyncDriver status: %d\n", st); @@ -208,7 +245,7 @@ VmBackupDriverOpCancel(VmBackupOp *_op) // IN static VmBackupDriverOp * VmBackupNewDriverOp(VmBackupState *state, // IN - Bool freeze, // IN + VmBackupOpType opType, // IN SyncDriverHandle *handle, // IN const char *volumes, // IN Bool useNullDriverPrefs) // IN @@ -216,8 +253,9 @@ VmBackupNewDriverOp(VmBackupState *state, // IN Bool success; VmBackupDriverOp *op = NULL; - g_return_val_if_fail((handle == NULL || *handle == SYNCDRIVER_INVALID_HANDLE) || - !freeze, + g_return_val_if_fail((handle == NULL || + *handle == SYNCDRIVER_INVALID_HANDLE) || + opType != OP_FREEZE, NULL); op = Util_SafeMalloc(sizeof *op); @@ -226,24 +264,32 @@ VmBackupNewDriverOp(VmBackupState *state, // IN op->callbacks.queryFn = VmBackupDriverOpQuery; op->callbacks.cancelFn = VmBackupDriverOpCancel; op->callbacks.releaseFn = VmBackupDriverOpRelease; - op->freeze = freeze; + op->opType = opType; op->volumes = volumes; op->syncHandle = g_new0(SyncDriverHandle, 1); *op->syncHandle = (handle != NULL) ? *handle : SYNCDRIVER_INVALID_HANDLE; - if (freeze) { - success = SyncDriver_Freeze(op->volumes, - useNullDriverPrefs ? - state->enableNullDriver : FALSE, - op->syncHandle, - state->excludedFileSystems); - } else { - op->manifest = SyncNewManifest(state, *op->syncHandle); - success = VmBackupDriverThaw(op->syncHandle); + switch (opType) { + case OP_FREEZE: + success = SyncDriver_Freeze(op->volumes, + useNullDriverPrefs ? + state->enableNullDriver : FALSE, + op->syncHandle, + state->excludedFileSystems); + break; + case OP_THAW: + op->manifest = SyncNewManifest(state, *op->syncHandle); + success = VmBackupDriverThaw(op->syncHandle); + break; + default: + ASSERT(opType == OP_UNDO); + success = VmBackupDriverThaw(op->syncHandle); + break; } if (!success) { - g_warning("Error %s filesystems.", freeze ? "freezing" : "thawing"); + g_warning("Error trying to perform %s on filesystems.", + VmBackupOpName[opType]); g_free(op->syncHandle); SyncManifestRelease(op->manifest); free(op); @@ -329,7 +375,7 @@ VmBackupSyncDriverStart(VmBackupState *state, VmBackupDriverOp *op; g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, TRUE, NULL, state->volumes, TRUE); + op = VmBackupNewDriverOp(state, OP_FREEZE, NULL, state->volumes, TRUE); if (op != NULL) { state->clientData = op->syncHandle; @@ -366,7 +412,7 @@ VmBackupSyncDriverOnlyStart(VmBackupState *state, VmBackupDriverOp *op; g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, TRUE, NULL, state->volumes, FALSE); + op = VmBackupNewDriverOp(state, OP_FREEZE, NULL, state->volumes, FALSE); if (op != NULL) { state->clientData = op->syncHandle; @@ -404,7 +450,7 @@ VmBackupSyncDriverStart(ToolsAppCtx *ctx, VmBackupState *state = (VmBackupState*) clientData; g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, TRUE, NULL, state->volumes, TRUE); + op = VmBackupNewDriverOp(state, OP_FREEZE, NULL, state->volumes, TRUE); if (op != NULL) { state->clientData = op->syncHandle; @@ -442,7 +488,7 @@ VmBackupSyncDriverOnlyStart(ToolsAppCtx *ctx, VmBackupState *state = (VmBackupState*) clientData; g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, TRUE, NULL, state->volumes, FALSE); + op = VmBackupNewDriverOp(state, OP_FREEZE, NULL, state->volumes, FALSE); if (op != NULL) { state->clientData = op->syncHandle; @@ -480,7 +526,7 @@ VmBackupSyncDriverSnapshotDone(VmBackupState *state, g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, FALSE, state->clientData, NULL, TRUE); + op = VmBackupNewDriverOp(state, OP_THAW, state->clientData, NULL, TRUE); g_free(state->clientData); state->clientData = NULL; @@ -513,12 +559,78 @@ VmBackupSyncDriverOnlySnapshotDone(VmBackupState *state, g_debug("*** %s\n", __FUNCTION__); - op = VmBackupNewDriverOp(state, FALSE, state->clientData, NULL, FALSE); + op = VmBackupNewDriverOp(state, OP_THAW, state->clientData, NULL, FALSE); + g_free(state->clientData); + state->clientData = NULL; + + return VmBackup_SetCurrentOp(state, (VmBackupOp *) op, NULL, __FUNCTION__); +} + + +#if defined(__linux__) +/* + *----------------------------------------------------------------------------- + * + * VmBackupSyncDriverUndo -- + * + * Undo a completed quiescing operation. + * + * Result + * TRUE, unless an error occurs. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +static Bool +VmBackupSyncDriverUndo(VmBackupState *state, + void *clientData) +{ + VmBackupDriverOp *op; + + g_debug("*** %s\n", __FUNCTION__); + + op = VmBackupNewDriverOp(state, OP_UNDO, state->clientData, NULL, TRUE); + g_free(state->clientData); + state->clientData = NULL; + + return VmBackup_SetCurrentOp(state, (VmBackupOp *) op, NULL, __FUNCTION__); +} + + +/* + *----------------------------------------------------------------------------- + * + * VmBackupSyncDriverOnlyUndo -- + * + * Undo a completed quiescing operation. + * + * Result + * TRUE, unless an error occurs. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +static Bool +VmBackupSyncDriverOnlyUndo(VmBackupState *state, + void *clientData) +{ + VmBackupDriverOp *op; + + g_debug("*** %s\n", __FUNCTION__); + + op = VmBackupNewDriverOp(state, OP_UNDO, state->clientData, NULL, FALSE); g_free(state->clientData); state->clientData = NULL; return VmBackup_SetCurrentOp(state, (VmBackupOp *) op, NULL, __FUNCTION__); } +#endif /* @@ -579,10 +691,17 @@ VmBackup_NewSyncDriverProviderInternal(Bool useNullDriverPrefs) if (useNullDriverPrefs) { provider->start = VmBackupSyncDriverStart; provider->snapshotDone = VmBackupSyncDriverSnapshotDone; +#if defined(__linux__) + provider->undo = VmBackupSyncDriverUndo; +#endif } else { provider->start = VmBackupSyncDriverOnlyStart; provider->snapshotDone = VmBackupSyncDriverOnlySnapshotDone; +#if defined(__linux__) + provider->undo = VmBackupSyncDriverOnlyUndo; +#endif } + provider->release = VmBackupSyncDriverRelease; provider->clientData = NULL; diff --git a/services/plugins/vmbackup/vmBackupInt.h b/services/plugins/vmbackup/vmBackupInt.h index 7b819ac15..ad3f2d7c2 100644 --- a/services/plugins/vmbackup/vmBackupInt.h +++ b/services/plugins/vmbackup/vmBackupInt.h @@ -156,6 +156,7 @@ typedef struct VmBackupSyncProvider { VmBackupProviderCallback start; #else ToolsCorePoolCb start; + VmBackupProviderCallback undo; #endif VmBackupProviderCallback snapshotDone; void (*release)(struct VmBackupSyncProvider *); diff --git a/services/plugins/vmbackup/stateMachine.c b/services/plugins/vmbackup/stateMachine.c index 14d08a77f..281181729 100644 --- a/services/plugins/vmbackup/stateMachine.c +++ b/services/plugins/vmbackup/stateMachine.c @@ -224,6 +224,7 @@ VmBackup_SendEvent(const char *event, if (gBackupState->keepAlive != NULL) { g_source_destroy(gBackupState->keepAlive); g_source_unref(gBackupState->keepAlive); + gBackupState->keepAlive = NULL; } msg = g_strdup_printf(VMBACKUP_PROTOCOL_EVENT_SET" %s %u %s", @@ -267,19 +268,27 @@ VmBackup_SendEvent(const char *event, &result, &resultLen); #endif - if (!success) { + if (success) { + ASSERT(gBackupState->keepAlive == NULL); + gBackupState->keepAlive = + g_timeout_source_new(VMBACKUP_KEEP_ALIVE_PERIOD / 2); + VMTOOLSAPP_ATTACH_SOURCE(gBackupState->ctx, + gBackupState->keepAlive, + VmBackupKeepAliveCallback, + NULL, + NULL); + } else { g_warning("Failed to send vmbackup event: %s, result: %s.\n", msg, result); + if (gBackupState->rpcState != VMBACKUP_RPC_STATE_IGNORE) { + g_debug("Changing rpcState from %d to %d\n", + gBackupState->rpcState, VMBACKUP_RPC_STATE_ERROR); + gBackupState->rpcState = VMBACKUP_RPC_STATE_ERROR; + } } vm_free(result); g_free(msg); - gBackupState->keepAlive = g_timeout_source_new(VMBACKUP_KEEP_ALIVE_PERIOD / 2); - VMTOOLSAPP_ATTACH_SOURCE(gBackupState->ctx, - gBackupState->keepAlive, - VmBackupKeepAliveCallback, - NULL, - NULL); return success; } @@ -440,6 +449,12 @@ VmBackupDoAbort(void) { g_debug("*** %s\n", __FUNCTION__); ASSERT(gBackupState != NULL); + + /* + * Once we abort the operation, we don't care about RPC state. + */ + gBackupState->rpcState = VMBACKUP_RPC_STATE_IGNORE; + if (gBackupState->machineState != VMBACKUP_MSTATE_SCRIPT_ERROR && gBackupState->machineState != VMBACKUP_MSTATE_SYNC_ERROR) { const char *eventMsg = "Quiesce aborted."; @@ -623,6 +638,17 @@ VmBackupAsyncCallback(void *clientData) if (opPending) { goto exit; } + + /* + * VMX state might have changed when we were processing + * currentOp. This is usually detected by failures in + * sending backup event to the host. + */ + if (gBackupState->rpcState == VMBACKUP_RPC_STATE_ERROR) { + g_warning("Aborting backup operation due to RPC errors."); + VmBackupDoAbort(); + goto exit; + } } switch (gBackupState->machineState) { @@ -958,6 +984,7 @@ VmBackupStartCommon(RpcInData *data, gBackupState->enableNullDriver = VMBACKUP_CONFIG_GET_BOOL(ctx->config, "enableNullDriver", TRUE); + gBackupState->rpcState = VMBACKUP_RPC_STATE_NORMAL; g_debug("Using quiesceApps = %d, quiesceFS = %d, allowHWProvider = %d," " execScripts = %d, scriptArg = %s, timeout = %u," diff --git a/services/plugins/vmbackup/vmBackupInt.h b/services/plugins/vmbackup/vmBackupInt.h index ad3f2d7c2..4258ee0ae 100644 --- a/services/plugins/vmbackup/vmBackupInt.h +++ b/services/plugins/vmbackup/vmBackupInt.h @@ -72,6 +72,12 @@ typedef enum { VMBACKUP_MSTATE_SYNC_ERROR } VmBackupMState; +typedef enum { + VMBACKUP_RPC_STATE_NORMAL, + VMBACKUP_RPC_STATE_ERROR, + VMBACKUP_RPC_STATE_IGNORE +} VmBackupRpcState; + /** * This is a "base struct" for asynchronous operations monitored by the * state machine. Each implementation should provide these three functions @@ -138,6 +144,7 @@ typedef struct VmBackupState { Bool vssBootableSystemState; Bool vssPartialFileSupport; Bool vssUseDefault; + VmBackupRpcState rpcState; } VmBackupState; typedef Bool (*VmBackupCallback)(VmBackupState *); diff --git a/services/plugins/vmbackup/stateMachine.c b/services/plugins/vmbackup/stateMachine.c index 281181729..d38187d1f 100644 --- a/services/plugins/vmbackup/stateMachine.c +++ b/services/plugins/vmbackup/stateMachine.c @@ -201,17 +201,19 @@ VmBackupPrivSendMsg(gchar *msg, * Sends a command to the VMX asking it to update VMDB about a new backup event. * This will restart the keep-alive timer. * + * As the name implies, does not abort the quiesce operation on failure. + * * @param[in] event The event to set. * @param[in] code Error code. - * @param[in] dest Error description. + * @param[in] desc Error description. * * @return TRUE on success. */ Bool -VmBackup_SendEvent(const char *event, - const uint32 code, - const char *desc) +VmBackup_SendEventNoAbort(const char *event, + const uint32 code, + const char *desc) { Bool success; char *result = NULL; @@ -280,11 +282,6 @@ VmBackup_SendEvent(const char *event, } else { g_warning("Failed to send vmbackup event: %s, result: %s.\n", msg, result); - if (gBackupState->rpcState != VMBACKUP_RPC_STATE_IGNORE) { - g_debug("Changing rpcState from %d to %d\n", - gBackupState->rpcState, VMBACKUP_RPC_STATE_ERROR); - gBackupState->rpcState = VMBACKUP_RPC_STATE_ERROR; - } } vm_free(result); g_free(msg); @@ -293,6 +290,36 @@ VmBackup_SendEvent(const char *event, } +/** + * Sends a command to the VMX asking it to update VMDB about a new backup event. + * This will restart the keep-alive timer. + * + * Aborts the quiesce operation on RPC failure. + * + * @param[in] event The event to set. + * @param[in] code Error code. + * @param[in] desc Error description. + * + * @return TRUE on success. + */ + +Bool +VmBackup_SendEvent(const char *event, + const uint32 code, + const char *desc) +{ + Bool success = VmBackup_SendEventNoAbort(event, code, desc); + + if (!success && gBackupState->rpcState != VMBACKUP_RPC_STATE_IGNORE) { + g_debug("Changing rpcState from %d to %d\n", + gBackupState->rpcState, VMBACKUP_RPC_STATE_ERROR); + gBackupState->rpcState = VMBACKUP_RPC_STATE_ERROR; + } + + return success; +} + + /** * Cleans up the backup state object and sends a "done" event to the VMX. */ @@ -1361,7 +1388,7 @@ VmBackupDumpState(gpointer src, /** - * Reset callback. + * Reset callback. Currently does nothing. * * @param[in] src The source object. Unused. * @param[in] ctx Unused. @@ -1373,7 +1400,7 @@ VmBackupReset(gpointer src, ToolsAppCtx *ctx, gpointer data) { - VmBackup_SyncDriverReset(); + } diff --git a/services/plugins/vmbackup/syncDriverOps.c b/services/plugins/vmbackup/syncDriverOps.c index 1dd98e323..d659f1b41 100644 --- a/services/plugins/vmbackup/syncDriverOps.c +++ b/services/plugins/vmbackup/syncDriverOps.c @@ -761,26 +761,3 @@ VmBackup_NewSyncDriverOnlyProvider(void) } #endif - - -/* - *----------------------------------------------------------------------------- - * - * VmBackup_SyncDriverReset -- - * - * Reset function - * - * Results: - * None. - * - * Side effects: - * Whatever are the side effects of what it calls. - * - *----------------------------------------------------------------------------- - */ - -void -VmBackup_SyncDriverReset(void) -{ - SyncManifestReset(); -} diff --git a/services/plugins/vmbackup/syncManifest.c b/services/plugins/vmbackup/syncManifest.c index 224f7e8aa..4586c4c34 100644 --- a/services/plugins/vmbackup/syncManifest.c +++ b/services/plugins/vmbackup/syncManifest.c @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2017-2018 VMware, Inc. All rights reserved. + * Copyright (C) 2017-2019 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -49,12 +49,6 @@ static const char syncManifestFmt[] = { */ static const char syncManifestSwitch[] = "enableXmlManifest"; -/* - * If TRUE, indicates that VMTools should try to generate the backup - * manifest and send it to VMX; if FALSE, it won't try to do so. - */ -static Bool gSyncManifestTrySend = TRUE; - /* *----------------------------------------------------------------------------- @@ -95,12 +89,6 @@ SyncNewManifest(VmBackupState *state, // IN return NULL; } - if (!gSyncManifestTrySend) { - g_debug("No backup manifest generated since previous" - " attempt to send one to host failed.\n"); - return NULL; - } - manifest = g_new0(SyncManifest, 1); manifest->path = g_strdup_printf("%s/%s", state->configDir, syncManifestName); @@ -173,37 +161,14 @@ SyncManifestSend(SyncManifest *manifest) // IN return FALSE; } - if (!VmBackup_SendEvent(VMBACKUP_EVENT_GENERIC_MANIFEST, - VMBACKUP_SUCCESS, manifest->path)) { - g_warning("Host doesn't appear to support backup manifests " - "for Linux guests.\n"); - gSyncManifestTrySend = FALSE; + if (!VmBackup_SendEventNoAbort(VMBACKUP_EVENT_GENERIC_MANIFEST, + VMBACKUP_SUCCESS, manifest->path)) { + /* VmBackup_SendEventNoAbort logs the error */ + g_info("Non-fatal error occurred while sending %s, continuing " + "with the operation", VMBACKUP_EVENT_GENERIC_MANIFEST); return FALSE; } g_debug("Backup manifest was sent successfully.\n"); return TRUE; } - - -/* - *----------------------------------------------------------------------------- - * - * SyncManifestReset -- - * - * Reset SyncManifest global state - * - * Results: - * None - * - * Side effects: - * None - * - *----------------------------------------------------------------------------- - */ - -void -SyncManifestReset(void) -{ - gSyncManifestTrySend = TRUE; -} diff --git a/services/plugins/vmbackup/syncManifest.h b/services/plugins/vmbackup/syncManifest.h index 9e4e9eb30..fd226adbd 100644 --- a/services/plugins/vmbackup/syncManifest.h +++ b/services/plugins/vmbackup/syncManifest.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2017-2018 VMware, Inc. All rights reserved. + * Copyright (C) 2017-2019 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -45,9 +45,6 @@ SyncManifestSend(SyncManifest *manifest); void SyncManifestRelease(SyncManifest *manifest); -void -SyncManifestReset(void); - #else /* !defined(__linux__) */ typedef void SyncManifest; @@ -55,7 +52,6 @@ typedef void SyncManifest; #define SyncNewManifest(s, h) (NULL) #define SyncManifestSend(m) (TRUE) #define SyncManifestRelease(m) ASSERT(m == NULL) -#define SyncManifestReset() #endif /* defined(__linux__) */ diff --git a/services/plugins/vmbackup/vmBackupInt.h b/services/plugins/vmbackup/vmBackupInt.h index 4258ee0ae..97fef2142 100644 --- a/services/plugins/vmbackup/vmBackupInt.h +++ b/services/plugins/vmbackup/vmBackupInt.h @@ -303,8 +303,11 @@ VmBackup_SendEvent(const char *event, const uint32 code, const char *desc); -void -VmBackup_SyncDriverReset(void); + +Bool +VmBackup_SendEventNoAbort(const char *event, + const uint32 code, + const char *desc); #endif /* _VMBACKUPINT_H_*/ diff --git a/services/plugins/vmbackup/syncManifest.c b/services/plugins/vmbackup/syncManifest.c index 4586c4c34..ddc995903 100644 --- a/services/plugins/vmbackup/syncManifest.c +++ b/services/plugins/vmbackup/syncManifest.c @@ -27,6 +27,7 @@ #include "syncDriver.h" #include "syncManifest.h" #include "vm_tools_version.h" +#include "vmware/tools/log.h" #include #include