Improve solution to #369

grycap · Sep 7, 2017 · c2ce250 · c2ce250
1 parent ea858ba
commit c2ce250
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 16 deletions.
diff --git a/IM/InfrastructureManager.py b/IM/InfrastructureManager.py
@@ -183,18 +183,11 @@ def _launch_group(sel_inf, deploy_group, cloud_id, cloud_list, concrete_systems,
                         for _ in range(deploy.vm_number):
                             launched_vms.append((False, "No username for deploy: " + deploy.id))
                     else:
-                        try:
-                            InfrastructureManager.logger.debug(
-                                "Launching %d VMs of type %s" % (deploy.vm_number, concrete_system.name))
-                            launched_vms = cloud.cloud.getCloudConnector(sel_inf).launch_with_retry(
-                                sel_inf, launch_radl, requested_radl, deploy.vm_number, auth, Config.MAX_VM_FAILS)
-                        except Exception as e:
-                            InfrastructureManager.logger.exception("Error launching some of the VMs: %s" % e)
-                            for _ in range(deploy.vm_number):
-                                launched_vms.append((False, "Error launching the VMs of type %s to cloud ID %s"
-                                                     " of type %s. Cloud Provider Error: %s" % (concrete_system.name,
-                                                                                                cloud.cloud.id,
-                                                                                                cloud.cloud.type, e)))
+                        InfrastructureManager.logger.debug(
+                            "Launching %d VMs of type %s" % (deploy.vm_number, concrete_system.name))
+                        launched_vms = cloud.cloud.getCloudConnector(sel_inf).launch_with_retry(
+                            sel_inf, launch_radl, requested_radl, deploy.vm_number, auth, Config.MAX_VM_FAILS,
+                            Config.DELAY_BETWEEN_VM_RETRIES)
 
                 # this must never happen ...
                 if len(launched_vms) < deploy.vm_number:

diff --git a/IM/config.py b/IM/config.py
@@ -92,6 +92,7 @@ class Config:
     INF_CACHE_TIME = None
     VMINFO_JSON = False
     VM_NUM_USE_CTXT_DIST = 30
+    DELAY_BETWEEN_VM_RETRIES = 5
 
 config = ConfigParser()
 config.read([Config.IM_PATH + '/../im.cfg', Config.IM_PATH +

diff --git a/IM/connectors/CloudConnector.py b/IM/connectors/CloudConnector.py
@@ -2,6 +2,7 @@
 import subprocess
 import shutil
 import tempfile
+import time
 
 
 class CloudConnector:
@@ -88,7 +89,7 @@ def launch(self, inf, radl, requested_radl, num_vm, auth_data):
 
         raise NotImplementedError("Should have implemented this")
 
-    def launch_with_retry(self, inf, radl, requested_radl, num_vm, auth_data, max_num):
+    def launch_with_retry(self, inf, radl, requested_radl, num_vm, auth_data, max_num, delay):
         """
         Launch a set of VMs to the Cloud provider with a set of retries in case of failure
 
@@ -99,6 +100,7 @@ def launch_with_retry(self, inf, radl, requested_radl, num_vm, auth_data, max_nu
         - num_vm(int): number of instances to deploy.
         - auth_data(Authentication): Authentication data to access cloud provider.
         - max_num: Number of retries.
+        - delay: a sleep time between retries
 
                 Returns: a list of tuples with the format (success, vm).
            - The first value is True if the operation finished successfully or false otherwise.
@@ -109,9 +111,18 @@ def launch_with_retry(self, inf, radl, requested_radl, num_vm, auth_data, max_nu
         res_err = {}
         retries = 0
         while len(res_ok) < num_vm and retries < max_num:
+            if retries != 0:
+                time.sleep(delay)
             retries += 1
             err_count = 0
-            for success, vm in self.launch(inf, radl, requested_radl, num_vm - len(res_ok), auth_data):
+            try:
+                vms = self.launch(inf, radl, requested_radl, num_vm - len(res_ok), auth_data)
+            except Exception as ex:
+                self.log_exception("Error launching some of the VMs")
+                vms = []
+                for _ in range(num_vm - len(res_ok)):
+                    vms.append((False, "Error: %s" % ex))
+            for success, vm in vms:
                 if success:
                     res_ok.append(vm)
                 else:

diff --git a/test/unit/connectors/Azure.py b/test/unit/connectors/Azure.py
@@ -199,7 +199,7 @@ def test_20_launch(self, save_data, credentials, network_client, compute_client,
 
         cclient.virtual_machines.create_or_update.side_effect = self.create_vm
 
-        res = azure_cloud.launch_with_retry(InfrastructureInfo(), radl, radl, 3, auth, 2)
+        res = azure_cloud.launch_with_retry(InfrastructureInfo(), radl, radl, 3, auth, 2, 0)
         self.assertEqual(len(res), 3)
         self.assertTrue(res[0][0])
         self.assertTrue(res[1][0])

diff --git a/test/unit/test_im_logic.py b/test/unit/test_im_logic.py
@@ -221,7 +221,7 @@ def test_inf_creation_addition_clouds(self):
 
     @patch("IM.connectors.OCCI.OCCICloudConnector")
     def test_inf_creation_errors(self, occi):
-        """Create infrastructure """
+        """Create infrastructure with errors"""
 
         radl = """"
             network publica (outbound = 'yes')
@@ -258,6 +258,24 @@ def test_inf_creation_errors(self, occi):
                       'Error, no concrete system to deploy: wn in cloud: ost. '
                       'Check if a correct image is being used', res)
 
+        # this case must fail with two errors, first the OCCI one
+        auth0 = Authentication([{'id': 'occi', 'type': 'OCCI', 'proxy': 'proxy',
+                                'host': 'http://localhost:443'},
+                                {'id': 'one', 'type': 'OpenNebula', 'username': 'user',
+                                'password': 'pass', 'host': 'localhost:2633'},
+                                {'type': 'InfrastructureManager', 'username': 'test',
+                                 'password': 'tests'}])
+        infID = IM.CreateInfrastructure(radl, auth0)
+        res = IM.GetInfrastructureState(infID, auth0)
+        self.assertEqual(res['state'], VirtualMachine.FAILED)
+        res = IM.GetInfrastructureContMsg(infID, auth0)
+        self.assertIn('VM 0:\nError launching the VMs of type front to cloud ID one of type OpenNebula. '
+                      'Attempt 1: Error: [Errno 111] Connection refused\nAttempt 2: Error: [Errno 111] '
+                      'Connection refused\nAttempt 3: Error: [Errno 111] Connection refused', res)
+        self.assertIn('VM 1:\nError launching the VMs of type wn to cloud ID one of type OpenNebula. '
+                      'Attempt 1: Error: [Errno 111] Connection refused\nAttempt 2: Error: [Errno 111] '
+                      'Connection refused\nAttempt 3: Error: [Errno 111] Connection refused', res)
+
         # this case must work OK
         auth0 = Authentication([{'id': 'dummy', 'type': 'Dummy'},
                                 {'type': 'InfrastructureManager', 'username': 'test',