Handle services in FAILED_DEPLOYING

src/server/client.rb

@@ -5,6 +5,26 @@ module ProvisionEngine
     #
     class CloudClient
 
+        SERVICE_STATES = [
+            'PENDING',
+            'DEPLOYING',
+            'RUNNING',
+            'UNDEPLOYING',
+            'WARNING',
+            'DONE',
+            'FAILED_UNDEPLOYING',
+            'FAILED_DEPLOYING',
+            'SCALING',
+            'FAILED_SCALING',
+            'COOLDOWN',
+            'DEPLOYING_NETS',
+            'UNDEPLOYING_NETS',
+            'FAILED_DEPLOYING_NETS',
+            'FAILED_UNDEPLOYING_NETS',
+            'HOLD'
+        ]
+
+
         def self.map_error_oned(xmlrpc_errno)
             # ESUCCESS        = 0x0000
             # EAUTHENTICATION = 0x0100
@@ -107,6 +127,13 @@ def service_delete(id)
             return_http_response(response)
         end
 
+        def service_recover(id, options = {})
+            @logger.debug("Forcing service #{id} deletion")
+
+            response = service_action(id, 'recover', options)
+            return_http_response(response)
+        end
+
         def service_template_get(id)
             response = @client_oneflow.get("/service_template/#{id}")
             return_http_response(response)
@@ -124,6 +151,14 @@ def service_template_instantiate(id, options = {})
             return_http_response(response)
         end
 
+        def service_fail?(service)
+            SERVICE_STATES[service_state(service)].include?('FAILED')
+        end
+
+        def service_state(service)
+            service['DOCUMENT']['TEMPLATE']['BODY']['state']
+        end
+
         private
 
         def return_http_response(response)
@@ -146,7 +181,7 @@ def service_template_action(id, action, options = {})
         end
 
         def service_action(id, action, options = {})
-            url = "/service/#{id}/action", body
+            url = "/service/#{id}/action"
 
             flow_element_action(url, action, options)
         end

src/server/runtime.rb

@@ -178,13 +178,16 @@ def self.create(client, specification)
             rc = response[0]
             rb = response[1]
 
-            return [rc, rb] if rc != 201
-
-            service_id = rb['DOCUMENT']['ID']
+            return [rc, rb] if rc != 200
 
+            service_id = rb['DOCUMENT']['ID'].to_i
+            specification['SERVICE_ID'] = service_id
             client.logger.info("#{SR} Service #{service_id} created")
 
-            ServerlessRuntime.service_sync(client, specification, service_id)
+            response = ServerlessRuntime.service_sync(client, specification, service_id)
+            rc = response[0]
+
+            return [rc, response[1]] if rc != 200
 
             client.logger.info("Allocating #{SR} Document")
             client.logger.debug(specification)
@@ -217,7 +220,10 @@ def self.get(client, id)
             runtime.load_body
             service_id = runtime.body['SERVICE_ID']
 
-            ServerlessRuntime.service_sync(client, runtime.body, service_id)
+            response = ServerlessRuntime.service_sync(client, runtime.body, service_id)
+            rc = response[0]
+            return [rc, response[1]] if rc != 200
+
             runtime.update
 
             [200, runtime]
@@ -256,6 +262,40 @@ def delete
             [204, '']
         end
 
+        def self.service_recover(client, service_id, options = {})
+            if options[:delete]
+                response = client.service_recover(service_id, { 'delete' => true })
+                rc = response[0]
+
+                return response if rc == 204
+
+                client.logger.error(response[1])
+                msg = "Could not force service #{service_id} deletion"
+                return [rc, msg]
+            else
+                response = client.service_recover(service_id)
+
+                if response[0] != 201
+                    client.logger.error("Could not recover service #{service_id}")
+                    return response
+                end
+
+                response = client.service_get(service_id)
+
+                return response unless response[0] == 200
+
+                service = response[1]
+
+                if client.service_fail?(service)
+                    msg = "Cannot recover #{service_id} from failure"
+                    client.logger.error(service)
+                    return [500, msg]
+                end
+
+                return [200, service]
+            end
+        end
+
         #
         # Validates the Serverless Runtime specification using the distributed schema
         #
@@ -316,48 +356,48 @@ def to_sr
         end
 
         #
-        # Updates Serverless Runtime Document specification based on the underlying elements state
+        # Updates Serverless Runtime definition based on the underlying elements state
         #
         # @param [CloudClient] client OpenNebula interface
-        # @param [Hash] runtime_definition Serverless Runtime definition to be updated
+        # @param [Hash] runtim Serverless Runtime definition to be updated
         # @param [Integer] service_id OneFlow service ID mapped to the Serverless Runtime
         # @param [Integer] timeout How long to wait for Role VMs to be created
         #
-        def self.service_sync(client, runtime_definition, service_id, timeout = 30)
+        def self.service_sync(client, runtime, service_id, timeout = 30)
             1.upto(timeout) do |t|
-                sleep 1
+                catch(:query_service) do
+                    if t == 30
+                        msg = "OpenNebula did not create VMs for the #{SR} service after #{t} seconds"
+                        return [504, msg]
+                    end
 
-                if t == 30
-                    msg = "OpenNebula did not create VMs for the #{SR} service after #{t} seconds"
-                    return [504, msg]
-                end
+                    sleep 1
 
-                response = client.service_get(service_id)
-                rc = response[0]
-                rb = response[1]
+                    response = client.service_get(service_id)
+                    rc = response[0]
+                    rb = response[1]
 
-                return [rc, rb] if rc != 200
+                    return [rc, rb] if rc != 200
 
-                service = rb
+                    roles = rb['DOCUMENT']['TEMPLATE']['BODY']['roles']
 
-                service_template = service['DOCUMENT']['TEMPLATE']['BODY']
-                roles = service_template['roles']
+                    roles.each do |role|
+                        next if role['nodes'].size < role['cardinality']
 
-                begin
-                    roles[0]['nodes'][0]['vm_info']['VM']
-                rescue NoMethodError # will fail if service VM information is missing
-                    client.logger.debug("Waiting #{t} seconds for service VMs")
+                        msg = "Waiting #{t} seconds for service role #{role['name']} VMs"
+                        client.logger.debug(msg)
 
-                    next
-                end
+                        throw(:query_service)
+                    end
 
-                client.logger.debug(service)
+                    client.logger.debug(service)
 
-                runtime_definition['SERVICE_ID'] = service['DOCUMENT']['ID'].to_i
-                runtime_definition['FAAS'].merge!(xaas_template(client, roles[0]))
-                runtime_definition['DAAS'].merge!(xaas_template(client, roles[1])) if roles[1]
+                    roles.each do |role|
+                        runtime[role['name']].merge!(xaas_template(client, role))
+                    end
 
-                break
+                    return [200, '']
+                end
             end
         end
 
@@ -436,7 +476,29 @@ def self.to_service(client, specification)
                     end
                 end
 
-                return client.service_template_instantiate(service_template['ID'], merge_template)
+                response = client.service_template_instantiate(service_template['ID'],
+                                                               merge_template)
+                rc = response[0]
+                rb = response[1]
+
+                return response if rc != 201
+
+                service_id = rb['DOCUMENT']['ID'].to_i
+
+                response = client.service_get(service_id)
+                rc = response[0]
+                rb = response[1]
+
+                return response if rc != 200
+
+                service = rb
+
+                if client.service_fail?(service)
+                    client.logger.error("#{SR} service #{service_id} entered FAILED state\n#{service}")
+                    response = service_recover(client, service_id, { 'delete' => true })
+                end
+
+                return response
             end
 
             msg = "Cannot find a valid service template for the specified flavours: #{tuple}\n"

src/server/server.rb

@@ -158,7 +158,7 @@ def log_response(level, code, data, message)
         log_response('error', rc, rb, "Timeout when creating #{SR}")
         halt rc, json_response(rc, rb)
     else
-        log_response('error', rc, rb, "Failed to create #{SR}")
+        log_response('error', 500, rb, "Failed to create #{SR}")
         halt 500, json_response(500, rb)
     end
 end
@@ -189,7 +189,7 @@ def log_response(level, code, data, message)
         log_response('error', rc, rb, SR_NOT_FOUND)
         halt rc, json_response(rc, rb)
     else
-        log_response('error', rc, rb, "Failed to get #{SR}")
+        log_response('error', 500, rb, "Failed to get #{SR}")
         halt 500, json_response(500, rb)
     end
 end
@@ -244,7 +244,7 @@ def log_response(level, code, data, message)
             log_response('error', rc, rb, NO_DELETE)
             halt rc, json_response(rc, rb)
         else
-            log_response('error', rc, rb, NO_DELETE)
+            log_response('error', 500, rb, NO_DELETE)
             halt 500, json_response(500, rb)
         end
     when 401
@@ -257,7 +257,7 @@ def log_response(level, code, data, message)
         log_response('error', rc, rb, SR_NOT_FOUND)
         halt rc, json_response(rc, rb)
     else
-        log_response('error', rc, rb, NO_DELETE)
+        log_response('error', 500, rb, NO_DELETE)
         halt 500, json_response(500, rb)
     end
 end

tests/lib/crud_invalid.rb

@@ -10,7 +10,7 @@
                 'FAAS' => {
                     'FLAVOUR' => 'Function'
                 },
-                'DAAS' => {}, # DAAS should be null or have properties
+                'DAAS' => {}, # DAAS should not exist or have FLAVOUR at least
                 'SCHEDULING' => {},
                 'DEVICE_INFO' => {}
             }