This is an automated email from the ASF dual-hosted git repository.
sureshanaparti pushed a commit to branch 4.19
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.19 by this push:
new 7f0d9a03045 [Veeam] Check for failures in the restore process (#7224)
7f0d9a03045 is described below
commit 7f0d9a03045af506ea9f37dc40eec45bbeeb174f
Author: SadiJr <[email protected]>
AuthorDate: Tue Jun 25 16:11:38 2024 -0300
[Veeam] Check for failures in the restore process (#7224)
* Validate failure state in Veeam restore process
* Address Daan review, and properly call method
* Address bryan's reviews
* remove return
Co-authored-by: SadiJr <[email protected]>
Co-authored-by: João Jandre <[email protected]>
---
.../cloudstack/backup/veeam/VeeamClient.java | 43 ++++++++++++++++++++--
.../cloudstack/backup/veeam/VeeamClientTest.java | 40 +++++++++++++++++++-
2 files changed, 78 insertions(+), 5 deletions(-)
diff --git
a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java
b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java
index 701c45f1a9d..befeb231015 100644
---
a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java
+++
b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java
@@ -345,7 +345,7 @@ public class VeeamClient {
String type = pair.second();
String path = url.replace(apiURI.toString(), "");
if (type.equals("RestoreSession")) {
- return checkIfRestoreSessionFinished(type, path);
+ checkIfRestoreSessionFinished(type, path);
}
}
return true;
@@ -361,17 +361,29 @@ public class VeeamClient {
return false;
}
- protected boolean checkIfRestoreSessionFinished(String type, String path)
throws IOException {
- for (int j = 0; j < this.restoreTimeout; j++) {
+
+ /**
+ * Checks the status of the restore session. Checked states are "Success"
and "Failure".<br/>
+ * There is also a timeout defined in the global configuration,
backup.plugin.veeam.restore.timeout,<br/>
+ * that is used to wait for the restore to complete before throwing a
{@link CloudRuntimeException}.
+ */
+ protected void checkIfRestoreSessionFinished(String type, String path)
throws IOException {
+ for (int j = 0; j < restoreTimeout; j++) {
HttpResponse relatedResponse = get(path);
RestoreSession session =
parseRestoreSessionResponse(relatedResponse);
if (session.getResult().equals("Success")) {
- return true;
+ return;
}
+
if (session.getResult().equalsIgnoreCase("Failed")) {
String sessionUid = session.getUid();
+ LOG.error(String.format("Failed to restore backup [%s] of VM
[%s] due to [%s].",
+ sessionUid, session.getVmDisplayName(),
+
getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":"))));
throw new CloudRuntimeException(String.format("Restore job
[%s] failed.", sessionUid));
}
+ LOG.debug(String.format("Waiting %s seconds, out of a total of %s
seconds, for the restore backup process to finish.", j, restoreTimeout));
+
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
@@ -930,6 +942,29 @@ public class VeeamClient {
return new Pair<>(result.first(), restoreLocation);
}
+ /**
+ * Tries to retrieve the error's description of the Veeam restore task
that resulted in an error.
+ * @param uid Session uid in Veeam of the restore process;
+ * @return the description found in Veeam about the cause of error in the
restore process.
+ */
+ protected String getRestoreVmErrorDescription(String uid) {
+ LOG.debug(String.format("Trying to find the cause of error in the
restore process [%s].", uid));
+ List<String> cmds = Arrays.asList(
+ String.format("$restoreUid = '%s'", uid),
+ "$restore = Get-VBRRestoreSession -Id $restoreUid",
+ "if ($restore) {",
+ "Write-Output $restore.Description",
+ "} else {",
+ "Write-Output 'Cannot find restore session with provided
uid $restoreUid'",
+ "}"
+ );
+ Pair<Boolean, String> result = executePowerShellCommands(cmds);
+ if (result != null && result.first()) {
+ return result.second();
+ }
+ return String.format("Failed to get the description of the failed
restore session [%s]. Please contact an administrator.", uid);
+ }
+
private boolean isLegacyServer() {
return this.veeamServerVersion != null && (this.veeamServerVersion > 0
&& this.veeamServerVersion < 11);
}
diff --git
a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java
b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java
index 06804d68da2..26b2449b0fe 100644
---
a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java
+++
b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java
@@ -58,6 +58,8 @@ public class VeeamClientTest {
private VeeamClient mockClient;
private static final SimpleDateFormat newDateFormat = new
SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private VeeamClient mock = Mockito.mock(VeeamClient.class);
+
@Rule
public WireMockRule wireMockRule = new WireMockRule(9399);
@@ -161,7 +163,7 @@ public class VeeamClientTest {
Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession);
Mockito.when(restoreSession.getResult()).thenReturn("No Success");
-
Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"),
Mockito.eq("any"))).thenCallRealMethod();
+
Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"),
Mockito.eq("any"));
mockClient.checkIfRestoreSessionFinished("RestoreTest", "any");
fail();
} catch (Exception e) {
@@ -170,6 +172,42 @@ public class VeeamClientTest {
Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
}
+ @Test
+ public void getRestoreVmErrorDescriptionTestFindErrorDescription() {
+ Pair<Boolean, String> response = new Pair<>(true, "Example of error
description found in Veeam.");
+
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+ String result = mock.getRestoreVmErrorDescription("uuid");
+ Assert.assertEquals("Example of error description found in Veeam.",
result);
+ }
+
+ @Test
+ public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() {
+ Pair<Boolean, String> response = new Pair<>(true, "Cannot find restore
session with provided uid uuid");
+
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+ String result = mock.getRestoreVmErrorDescription("uuid");
+ Assert.assertEquals("Cannot find restore session with provided uid
uuid", result);
+ }
+
+ @Test
+ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() {
+
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null);
+ String result = mock.getRestoreVmErrorDescription("uuid");
+ Assert.assertEquals("Failed to get the description of the failed
restore session [uuid]. Please contact an administrator.", result);
+ }
+
+ @Test
+ public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() {
+ Pair<Boolean, String> response = new Pair<>(false, null);
+
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
+
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
+ String result = mock.getRestoreVmErrorDescription("uuid");
+ Assert.assertEquals("Failed to get the description of the failed
restore session [uuid]. Please contact an administrator.", result);
+ }
+
+
private void verifyBackupMetrics(Map<String, Backup.Metric> metrics) {
Assert.assertEquals(2, metrics.size());
