[Veeam] externalize restore timeout (apache#6320)

* [Veeam] add global timeout configuration for backup restore process

* Use 'this'

* Address reviews

* Address reviews

Co-authored-by: SadiJr <[email protected]>

Author: SadiJr

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java

@@ -77,6 +77,9 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
     private ConfigKey<Integer> VeeamApiRequestTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.request.timeout", "300",
             "The Veeam B&R API request timeout in seconds.", true, ConfigKey.Scope.Zone);
 
+    private static ConfigKey<Integer> VeeamRestoreTimeout = new ConfigKey<>("Advanced", Integer.class, "backup.plugin.veeam.restore.timeout", "600",
+            "The Veeam B&R API restore backup timeout in seconds.", true, ConfigKey.Scope.Zone);
+
     @Inject
     private VmwareDatacenterZoneMapDao vmwareDatacenterZoneMapDao;
     @Inject
@@ -87,7 +90,7 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
     private VeeamClient getClient(final Long zoneId) {
         try {
             return new VeeamClient(VeeamUrl.valueIn(zoneId), VeeamUsername.valueIn(zoneId), VeeamPassword.valueIn(zoneId),
-                VeeamValidateSSLSecurity.valueIn(zoneId), VeeamApiRequestTimeout.valueIn(zoneId));
+                    VeeamValidateSSLSecurity.valueIn(zoneId), VeeamApiRequestTimeout.valueIn(zoneId), VeeamRestoreTimeout.valueIn(zoneId));
         } catch (URISyntaxException e) {
             throw new CloudRuntimeException("Failed to parse Veeam API URL: " + e.getMessage());
         } catch (NoSuchAlgorithmException | KeyManagementException e) {
@@ -318,7 +321,8 @@ public ConfigKey<?>[] getConfigKeys() {
                 VeeamUsername,
                 VeeamPassword,
                 VeeamValidateSSLSecurity,
-                VeeamApiRequestTimeout
+                VeeamApiRequestTimeout,
+                VeeamRestoreTimeout
         };
     }
 

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java

@@ -94,10 +94,13 @@ public class VeeamClient {
     private String veeamServerUsername;
     private String veeamServerPassword;
     private String veeamSessionId = null;
+    private int restoreTimeout;
     private final int veeamServerPort = 22;
 
-    public VeeamClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException {
+    public VeeamClient(final String url, final String username, final String password, final boolean validateCertificate, final int timeout,
+            final int restoreTimeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException {
         this.apiURI = new URI(url);
+        this.restoreTimeout = restoreTimeout;
 
         final RequestConfig config = RequestConfig.custom()
                 .setConnectTimeout(timeout * 1000)
@@ -173,7 +176,7 @@ private void checkResponseTimeOut(final Exception e) {
         }
     }
 
-    private HttpResponse get(final String path) throws IOException {
+    protected HttpResponse get(final String path) throws IOException {
         String url = apiURI.toString() + path;
         final HttpGet request = new HttpGet(url);
         request.setHeader(SESSION_HEADER, veeamSessionId);
@@ -274,7 +277,7 @@ private Task parseTaskResponse(HttpResponse response) throws IOException {
         return objectMapper.readValue(response.getEntity().getContent(), Task.class);
     }
 
-    private RestoreSession parseRestoreSessionResponse(HttpResponse response) throws IOException {
+    protected RestoreSession parseRestoreSessionResponse(HttpResponse response) throws IOException {
         checkResponseOK(response);
         final ObjectMapper objectMapper = new XmlMapper();
         return objectMapper.readValue(response.getEntity().getContent(), RestoreSession.class);
@@ -297,18 +300,7 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
                         String type = pair.second();
                         String path = url.replace(apiURI.toString(), "");
                         if (type.equals("RestoreSession")) {
-                            for (int j = 0; j < 120; j++) {
-                                HttpResponse relatedResponse = get(path);
-                                RestoreSession session = parseRestoreSessionResponse(relatedResponse);
-                                if (session.getResult().equals("Success")) {
-                                    return true;
-                                }
-                                try {
-                                    Thread.sleep(5000);
-                                } catch (InterruptedException ignored) {
-                                }
-                            }
-                            throw new CloudRuntimeException("Related job type: " + type + " was not successful");
+                            return checkIfRestoreSessionFinished(type, path);
                         }
                     }
                     return true;
@@ -324,6 +316,22 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
         return false;
     }
 
+    protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException {
+        for (int j = 0; j < this.restoreTimeout; j++) {
+            HttpResponse relatedResponse = get(path);
+            RestoreSession session = parseRestoreSessionResponse(relatedResponse);
+            if (session.getResult().equals("Success")) {
+                return true;
+            }
+            try {
+                Thread.sleep(1000);
+            } catch (InterruptedException ignored) {
+                LOG.trace(String.format("Ignoring InterruptedException [%s] when waiting for restore session finishes.", ignored.getMessage()));
+            }
+        }
+        throw new CloudRuntimeException("Related job type: " + type + " was not successful");
+    }
+
     private Pair<String, String> getRelatedLinkPair(List<Link> links) {
         for (Link link : links) {
             if (link.getRel().equals("Related")) {

plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java

@@ -25,15 +25,20 @@
 import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.times;
 
+import java.io.IOException;
 import java.util.List;
 
 import org.apache.cloudstack.backup.BackupOffering;
+import org.apache.cloudstack.backup.veeam.api.RestoreSession;
+import org.apache.http.HttpResponse;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.Mockito;
+import org.springframework.test.util.ReflectionTestUtils;
 
 import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -57,7 +62,7 @@ public void setUp() throws Exception {
                         .withStatus(201)
                         .withHeader("X-RestSvcSessionId", "some-session-auth-id")
                         .withBody("")));
-        client = new VeeamClient("http://localhost:9399/api/", adminUsername, adminPassword, true, 60);
+        client = new VeeamClient("http://localhost:9399/api/", adminUsername, adminPassword, true, 60, 600);
         mockClient = Mockito.mock(VeeamClient.class);
         Mockito.when(mockClient.getRepositoryNameFromJob(Mockito.anyString())).thenCallRealMethod();
     }
@@ -139,4 +144,22 @@ public void getRepositoryNameFromJobTestSuccess() throws Exception {
         String repositoryNameFromJob = mockClient.getRepositoryNameFromJob(backupName);
         Assert.assertEquals("test", repositoryNameFromJob);
     }
+
+    @Test
+    public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOException {
+        try {
+            ReflectionTestUtils.setField(mockClient, "restoreTimeout", 10);
+            RestoreSession restoreSession = Mockito.mock(RestoreSession.class);
+            HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
+            Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
+            Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession);
+            Mockito.when(restoreSession.getResult()).thenReturn("No Success");
+            Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod();
+            mockClient.checkIfRestoreSessionFinished("RestoreTest", "any");
+            fail();
+        } catch (Exception e) {
+            Assert.assertEquals("Related job type: RestoreTest was not successful", e.getMessage());
+        }
+        Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
+    }
 }