Handle services in FAILED_DEPLOYING

[dann1]

When the flow service backing the SR fails to instantiate now

• there is an automatic oneflow recover --delete equivalent REST API Call issued to oneflow
• the SR document will not be created
• the failure will be fast

Response with error 500

 ~/P/C/provisioning-engine   flow_fail *$  src/client  ./http_cli.rb post http://localhost:1337/serverless-runtimes ../../tests/templates/sr_faas_FAILED_DEPLOY.json
{
  "error": "Serverless Runtime service 1065 entered FAILED state",
  "message": [
    {
      "timestamp": 1699916942,
      "severity": "I",
      "message": "New state: DEPLOYING_NETS"
    },
    {
      "timestamp": 1699916942,
      "severity": "E",
      "message": "Role FAAS : Instantiate failed for template 7; [one.template.instantiate] Error allocating a new virtual machine template. Cannot get IP/MAC lease from virtual network 1."
    },
    {
      "timestamp": 1699916942,
      "severity": "I",
      "message": "New state: FAILED_DEPLOYING"
    }
  ]
}

Logs

 /p/v/l/provision-engine  cat engine.log                                                                                                                                     8s
# Logfile created on 2023-11-13 17:09:00 -0600 by logger.rb/v1.5.3
I, [2023-11-13 17:09:00 #39449]  INFO -- : Initializing Provision Engine component: engine
I, [2023-11-13 17:09:00 #39449]  INFO -- : Using oned at http://localhost:1338/RPC2
I, [2023-11-13 17:09:00 #39449]  INFO -- : Using oneflow at http://localhost:1339
D, [2023-11-13 17:09:01 #39449] DEBUG -- : API Call: POST /serverless-runtimes {"SERVERLESS_RUNTIME":{"FAAS":{"FLAVOUR":"FAILED_DEPLOY"},"SCHEDULING":{},"DEVICE_INFO":{}}}
I, [2023-11-13 17:09:01 #39449]  INFO -- : Received request to Create a Serverless Runtime
I, [2023-11-13 17:09:03 #39449]  INFO -- : Response HTTP Return Code: 500
E, [2023-11-13 17:09:03 #39449] ERROR -- : Failed to create Serverless Runtime
D, [2023-11-13 17:09:03 #39449] DEBUG -- : Response Body: {"error":"Serverless Runtime service 1065 entered FAILED state","message":[{"timestamp":1699916942,"severity":"I","message":"New state: DEPLOYING_NETS"},{"timestamp":1699916942,"severity":"E","message":"Role FAAS : Instantiate failed for template 7; [one.template.instantiate] Error allocating a new virtual machine template. Cannot get IP/MAC lease from virtual network 1."},{"timestamp":1699916942,"severity":"I","message":"New state: FAILED_DEPLOYING"}]}
D, [2023-11-13 17:10:49 #39449] DEBUG -- : API Call: POST /serverless-runtimes {"SERVERLESS_RUNTIME":{"FAAS":{"FLAVOUR":"Function"},"SCHEDULING":{},"DEVICE_INFO":{}}}
I, [2023-11-13 17:10:49 #39449]  INFO -- : Received request to Create a Serverless Runtime
I, [2023-11-13 17:10:53 #39449]  INFO -- : Response HTTP Return Code: 201
I, [2023-11-13 17:10:53 #39449]  INFO -- : Serverless Runtime created
D, [2023-11-13 17:10:53 #39449] DEBUG -- : Response Body: {
  "DOCUMENT": {
    "ID": "1067",
    "UID": "0",
    "GID": "0",
    "UNAME": "oneadmin",
    "GNAME": "oneadmin",
    "NAME": "Function_bf26ed7a-001e-4239-bccb-525e2960770c",
    "TYPE": "1337",
    "PERMISSIONS": {
      "OWNER_U": "1",
      "OWNER_M": "1",
      "OWNER_A": "0",
      "GROUP_U": "0",
      "GROUP_M": "0",
      "GROUP_A": "0",
      "OTHER_U": "0",
      "OTHER_M": "0",
      "OTHER_A": "0"
    },
    "TEMPLATE": {
      "BODY": {
        "FAAS": {
          "FLAVOUR": "Function",
          "VM_ID": 715,
          "STATE": "PENDING",
          "ENDPOINT": null,
          "CPU": 0.1,
          "VCPU": 0,
          "MEMORY": 128,
          "DISK_SIZE": 200
        },
        "SCHEDULING": {
        },
        "DEVICE_INFO": {
        },
        "SERVICE_ID": 1066,
        "registration_time": 1699917052
      }
    }
  }
}
 /p/v/l/provision-engine  cat CloudClient.log
# Logfile created on 2023-11-13 17:10:49 -0600 by logger.rb/v1.5.3
I, [2023-11-13 17:10:49 #39449]  INFO -- : Initializing Provision Engine component: CloudClient
I, [2023-11-13 17:10:49 #39449]  INFO -- : Creating oneflow Service for Serverless Runtime
I, [2023-11-13 17:10:50 #39449]  INFO -- : Requesting Custom VM Requirements for function FAAS
{"FLAVOUR"=>"Function"}
D, [2023-11-13 17:10:50 #39449] DEBUG -- : Applying vm_template_contents to role FAAS
HOT_RESIZE=[CPU_HOT_ADD_ENABLED="YES",
MEMORY_HOT_ADD_ENABLED="YES"]
MEMORY_RESIZE_MODE="BALLOONING"
VCPU_MAX= "4"
MEMORY_MAX="256"
D, [2023-11-13 17:10:50 #39449] DEBUG -- : Instantiating service_template 241 with options {"roles"=>[{"name"=>"FAAS", "vm_template_contents"=>"HOT_RESIZE=[CPU_HOT_ADD_ENABLED=\"YES\",\nMEMORY_HOT_ADD_ENABLED=\"YES\"]\nMEMORY_RESIZE_MODE=\"BALLOONING\"\nVCPU_MAX= \"4\"\nMEMORY_MAX=\"256\"\nSCHEDULING=[]\nDEVICE_INFO=[]\n"}]}
I, [2023-11-13 17:10:51 #39449]  INFO -- : Serverless Runtime Service 1066 created
D, [2023-11-13 17:10:52 #39449] DEBUG -- : {"DOCUMENT"=>{"ID"=>"1066", "UID"=>"0", "GID"=>"0", "UNAME"=>"oneadmin", "GNAME"=>"oneadmin", "NAME"=>"Function", "TYPE"=>"100", "PERMISSIONS"=>{"OWNER_U"=>"1", "OWNER_M"=>"1", "OWNER_A"=>"0", "GROUP_U"=>"0", "GROUP_M"=>"0", "GROUP_A"=>"0", "OTHER_U"=>"0", "OTHER_M"=>"0", "OTHER_A"=>"0"}, "TEMPLATE"=>{"BODY"=>{"name"=>"Function", "deployment"=>"straight", "description"=>"", "roles"=>[{"name"=>"FAAS", "cardinality"=>1, "vm_template"=>5, "shutdown_action"=>"terminate-hard", "elasticity_policies"=>[], "scheduled_policies"=>[], "vm_template_contents"=>"HOT_RESIZE=[CPU_HOT_ADD_ENABLED=\"YES\",\nMEMORY_HOT_ADD_ENABLED=\"YES\"]\nMEMORY_RESIZE_MODE=\"BALLOONING\"\nVCPU_MAX= \"4\"\nMEMORY_MAX=\"256\"\nSCHEDULING=[]\nDEVICE_INFO=[]\n", "state"=>1, "cooldown"=>300, "nodes"=>[{"deploy_id"=>715, "vm_info"=>{"VM"=>{"ID"=>"715", "UID"=>"0", "GID"=>"0", "UNAME"=>"oneadmin", "GNAME"=>"oneadmin", "NAME"=>"FAAS_0_(service_1066)"}}}], "on_hold"=>false, "last_vmname"=>1}], "ready_status_gate"=>false, "automatic_deletion"=>false, "registration_time"=>1696454194, "state"=>1, "start_time"=>1699917051, "log"=>[{"timestamp"=>1699917051, "severity"=>"I", "message"=>"New state: DEPLOYING_NETS"}, {"timestamp"=>1699917051, "severity"=>"I", "message"=>"New state: DEPLOYING"}]}}}}
I, [2023-11-13 17:10:52 #39449]  INFO -- : Allocating Serverless Runtime Document
D, [2023-11-13 17:10:52 #39449] DEBUG -- : {"FAAS"=>{"FLAVOUR"=>"Function", "VM_ID"=>715, "STATE"=>"PENDING", "ENDPOINT"=>nil, "CPU"=>0.1, "VCPU"=>0, "MEMORY"=>128, "DISK_SIZE"=>200}, "SCHEDULING"=>{}, "DEVICE_INFO"=>{}, "SERVICE_ID"=>1066}
I, [2023-11-13 17:10:53 #39449]  INFO -- : Created Serverless Runtime Document

Still missing tests