From b69aeed4f60f680b9fcc87aa07b92d233d766b9f Mon Sep 17 00:00:00 2001 From: Daniel Clavijo Coca Date: Fri, 2 Feb 2024 19:36:00 -0600 Subject: [PATCH 1/3] More robust SCHEDULING and FLAVOUR saved on VM.USER_TEMPLATE --- share/api.yaml | 2 +- src/server/function.rb | 4 +++- src/server/server.rb | 3 ++- tests/conf.yaml | 8 ++++---- tests/lib/common.rb | 22 ++++++++++++---------- tests/templates/_sr_sched.json | 12 ++++++++---- 6 files changed, 30 insertions(+), 21 deletions(-) diff --git a/share/api.yaml b/share/api.yaml index bd202c3..b0c4c3c 100644 --- a/share/api.yaml +++ b/share/api.yaml @@ -1,6 +1,6 @@ openapi: "3.0.0" info: - version: "1.1.0" + version: "1.2.0" title: "Provisioning Engine REST API" description: Provides FaaS capabilities by leveraging features from OpenNebula. Allows to manage Serverless Runtime instances based on a group of Functions defined on request. diff --git a/src/server/function.rb b/src/server/function.rb index 4960b10..6dc8ba9 100644 --- a/src/server/function.rb +++ b/src/server/function.rb @@ -327,7 +327,7 @@ def self.map_user_template(specification) if specification.key?('SCHEDULING') specification['SCHEDULING'].each do |property, value| - schevice << "#{Function::SCHED_MAP[property]}=\"#{value}\"\n" if value + schevice << "#{Function::SCHED_MAP[property]}=\"#{value.upcase}\"\n" if value end end @@ -346,6 +346,8 @@ def self.map_user_template(specification) schevice << "DEVICE_INFO=[#{i_template}]\n" end + schevice << "FLAVOURS=\"#{ProvisionEngine::ServerlessRuntime.tuple(specification)}\"\n" + schevice end diff --git a/src/server/server.rb b/src/server/server.rb index 9fd61ed..6dc4cff 100644 --- a/src/server/server.rb +++ b/src/server/server.rb @@ -25,11 +25,12 @@ require 'runtime' require 'function' +VERSION = '1.2.0' + ############################################################################ # API configuration ############################################################################ -VERSION = '1.1.0' conf = ProvisionEngine::Configuration.new configure do diff --git a/tests/conf.yaml b/tests/conf.yaml index 4e7f293..df3ff81 100644 --- a/tests/conf.yaml +++ b/tests/conf.yaml @@ -3,8 +3,8 @@ :delete: 30 :examples: # What group of examples should be tested 'crud': true - 'auth': true - 'crud_invalid': true - 'server_info': true - 'logs': true + 'auth': false + 'crud_invalid': false + 'server_info': false + 'logs': false :purge: true # delete services owned by the user running the tests. DO NOT RUN AS ONEADMIN !!! diff --git a/tests/lib/common.rb b/tests/lib/common.rb index e9e3cc6..d98719e 100644 --- a/tests/lib/common.rb +++ b/tests/lib/common.rb @@ -88,19 +88,21 @@ def verify_sr_spec(specification, runtime) expect(runtime[role]['ERROR']).to eq(vm["#{T}ERROR"]) end - ['DEVICE_INFO', 'SCHEDULING'].each do |schevice| - next unless specification.key?(schevice) + # Verify VM.USER_TEMPLATE - if schevice == 'SCHEDULING' - specification[schevice].each do |k, v| - expect(vm["#{UT}#{ProvisionEngine::Function::SCHED_MAP[k]}"]).to eq(v.to_s) - end - else - specification[schevice].each do |k, v| - expect(vm["#{UT}#{schevice}/#{k}"]).to eq(v.to_s) - end + expect(vm["#{UT}FLAVOURS"]).to eq(ProvisionEngine::ServerlessRuntime.tuple(specification)) + + if specification.key?('SCHEDULING') + specification['SCHEDULING'].each do |k, v| + expect(vm["#{UT}#{ProvisionEngine::Function::SCHED_MAP[k]}"]).to eq(v.to_s) end end + + next unless specification.key?('DEVICE_INFO') + + specification['DEVICE_INFO'].each do |k, v| + expect(vm["#{UT}DEVICE_INFO/#{k}"]).to eq(v.to_s) + end end end diff --git a/tests/templates/_sr_sched.json b/tests/templates/_sr_sched.json index 8b77596..1510533 100644 --- a/tests/templates/_sr_sched.json +++ b/tests/templates/_sr_sched.json @@ -1,15 +1,19 @@ { "SERVERLESS_RUNTIME": { + "NAME": "Example Serverless Runtime", "FAAS": { - "FLAVOUR": "Function" + "CPU": 1, + "MEMORY": 768, + "DISK_SIZE": 3072, + "FLAVOUR": "Function", + "ENDPOINT": "" }, "SCHEDULING": { "POLICY": "energy", - "REQUIREMENTS": "FREECPU > 10" + "REQUIREMENTS": "energy=50" }, "DEVICE_INFO": { - "LATENCY_TO_PE": 100, - "GEOGRAPHIC_LOCATION": "Madrid" + "GEOGRAPHIC_LOCATION": "{'ip':'193.145.247.253','city':'Arrasate / Mondragón','region':'Basque Country','country':'ES','loc':'43.0644,-2.4898','org':'AS766 Entidad Publica Empresarial Red.es','postal':'20500','timezone':'Europe/Madrid','readme':'https://ipinfo.io/missingauth'}" } } } From 02925c26fb2a3f645c8bb8a9691d6f2f918a1c1a Mon Sep 17 00:00:00 2001 From: Daniel Clavijo Coca Date: Tue, 6 Feb 2024 17:29:59 -0600 Subject: [PATCH 2/3] Fix tuple handling and Improved UT mapping --- src/server/function.rb | 72 ++++++++++++++++++++---------------------- src/server/runtime.rb | 21 ++++++------ 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/src/server/function.rb b/src/server/function.rb index 6dc8ba9..0927b6f 100644 --- a/src/server/function.rb +++ b/src/server/function.rb @@ -143,7 +143,7 @@ def self.get(client, id) # # @return [String] vm_template contents for a oneflow service template # - def self.vm_template_contents(specification, vm_template, conf_capacity) + def self.map_vm_template(specification, vm_template, conf_capacity) disk_size = specification['DISK_SIZE'] disk_size ||= conf_capacity[:disk][:default] @@ -184,6 +184,40 @@ def self.vm_template_contents(specification, vm_template, conf_capacity) xaas.join("\n") end + # + # Translates specification parameters to Function VM USER_TEMPLATE + # + # @param [Hash] specification Serverless Runtime definition + # + # @return [String] User Template string compatible with opennebula VM Template + # + def self.map_user_template(specification) + user_template = '' + + if specification.key?('SCHEDULING') + specification['SCHEDULING'].each do |property, value| + user_template << "#{Function::SCHED_MAP[property]}=\"#{value.upcase}\"\n" if value + end + end + + if specification.key?('DEVICE_INFO') + i_template = '' + + specification['DEVICE_INFO'].each do |property, value| + i_template << "#{property}=\"#{value}\",\n" if value + end + + if !i_template.empty? + i_template.reverse!.sub!("\n", '').reverse! + i_template.reverse!.sub!(',', '').reverse! + end + + user_template << "DEVICE_INFO=[#{i_template}]\n" + end + + user_template + end + # # Creates a runtime function hash for the Serverless Runtime document # @@ -315,42 +349,6 @@ def state_function return STATES[:updating] end - # - # Translates specification parameters to Function VM USER_TEMPLATE - # - # @param [Hash] specification Serverless Runtime definition - # - # @return [String] User Template string compatible with opennebula VM Template - # - def self.map_user_template(specification) - schevice='' - - if specification.key?('SCHEDULING') - specification['SCHEDULING'].each do |property, value| - schevice << "#{Function::SCHED_MAP[property]}=\"#{value.upcase}\"\n" if value - end - end - - if specification.key?('DEVICE_INFO') - i_template = '' - - specification['DEVICE_INFO'].each do |property, value| - i_template << "#{property}=\"#{value}\",\n" if value - end - - if !i_template.empty? - i_template.reverse!.sub!("\n", '').reverse! - i_template.reverse!.sub!(',', '').reverse! - end - - schevice << "DEVICE_INFO=[#{i_template}]\n" - end - - schevice << "FLAVOURS=\"#{ProvisionEngine::ServerlessRuntime.tuple(specification)}\"\n" - - schevice - end - end end diff --git a/src/server/runtime.rb b/src/server/runtime.rb index ea8f710..24bf667 100644 --- a/src/server/runtime.rb +++ b/src/server/runtime.rb @@ -212,13 +212,12 @@ def update_sr(specification) end - schevice = Function.map_user_template(specification) - - response = vm.update(schevice, true) unless schevice.empty? + user_template = Function.map_user_template(specification) + response = vm.update(user_template, true) unless user_template.empty? if OpenNebula.is_error?(response) rc = ProvisionEngine::Error.map_error_oned(response.errno) - error = "Failed to update #{SRF} #{schevice}" + error = "Failed to update #{SRF} #{user_template}" return ProvisionEngine::Error.new(rc, error, response.message) end @@ -390,10 +389,12 @@ def self.to_service(client, specification) service_templates.each do |service_template| next unless service_template['NAME'] == tuple + client.logger.info("Found matching flow template for tuple: #{tuple}") + merge_template = { 'roles' => [] } - schevice = Function.map_user_template(specification) + user_template = Function.map_user_template(specification) + "FLAVOURS=\"#{tuple}\"" ProvisionEngine::Function::FUNCTIONS.each do |role| next unless specification[role] && !specification[role]['FLAVOUR'].empty? @@ -418,15 +419,17 @@ def self.to_service(client, specification) return ProvisionEngine::Error.new(500, error) end - override = ProvisionEngine::Function.vm_template_contents(specification[role], vm_template, + vm_template = ProvisionEngine::Function.map_vm_template(specification[role], vm_template, client.conf[:capacity]) + vm_template_contents = "#{vm_template}\n#{user_template}" + client.logger.info("Applying \"vm_template_contents\" to role #{role}") - client.logger.debug(override) + client.logger.debug(vm_template_contents) merge_template['roles'] << { 'name' => role, - 'vm_template_contents' => "#{override}\n#{schevice}" + 'vm_template_contents' => vm_template_contents } end end @@ -597,7 +600,7 @@ def service_id # @return [String] FaasName possibly + -DaasName if DaaS flavour is specified # def self.tuple(specification) - tuple = specification['FAAS']['FLAVOUR'] + tuple = specification['FAAS']['FLAVOUR'].dup if specification['DAAS'] && !specification['DAAS']['FLAVOUR'].empty? tuple << "-#{specification['DAAS']['FLAVOUR']}" From 785764a06b5670d7a74416af0d3d533e37d5f437 Mon Sep 17 00:00:00 2001 From: Daniel Clavijo Coca Date: Tue, 6 Feb 2024 17:30:43 -0600 Subject: [PATCH 3/3] Fixed scheduling tests --- tests/lib/common.rb | 2 +- tests/lib/crud.rb | 5 ----- tests/templates/_sr_sched.json | 11 ++--------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/tests/lib/common.rb b/tests/lib/common.rb index d98719e..227e2a5 100644 --- a/tests/lib/common.rb +++ b/tests/lib/common.rb @@ -94,7 +94,7 @@ def verify_sr_spec(specification, runtime) if specification.key?('SCHEDULING') specification['SCHEDULING'].each do |k, v| - expect(vm["#{UT}#{ProvisionEngine::Function::SCHED_MAP[k]}"]).to eq(v.to_s) + expect(vm["#{UT}#{ProvisionEngine::Function::SCHED_MAP[k]}"]).to eq(v.to_s.upcase) end end diff --git a/tests/lib/crud.rb b/tests/lib/crud.rb index b6f6d7a..76e8ec7 100644 --- a/tests/lib/crud.rb +++ b/tests/lib/crud.rb @@ -54,8 +54,6 @@ case rc when 200 - pp runtime - verify_sr_spec(@conf[:runtime], runtime) break when 423 @@ -65,7 +63,6 @@ sleep 1 next else - pp runtime verify_error(runtime) raise "Unexpected error code #{rc}" @@ -81,8 +78,6 @@ runtime = JSON.parse(response.body) - pp runtime - verify_sr_spec(@conf[:runtime], runtime) end diff --git a/tests/templates/_sr_sched.json b/tests/templates/_sr_sched.json index 1510533..07a7730 100644 --- a/tests/templates/_sr_sched.json +++ b/tests/templates/_sr_sched.json @@ -1,19 +1,12 @@ { "SERVERLESS_RUNTIME": { - "NAME": "Example Serverless Runtime", + "NAME": "Example Serverless Runtime with defined SCHEDULING", "FAAS": { - "CPU": 1, - "MEMORY": 768, - "DISK_SIZE": 3072, - "FLAVOUR": "Function", - "ENDPOINT": "" + "FLAVOUR": "Function" }, "SCHEDULING": { "POLICY": "energy", "REQUIREMENTS": "energy=50" - }, - "DEVICE_INFO": { - "GEOGRAPHIC_LOCATION": "{'ip':'193.145.247.253','city':'Arrasate / Mondragón','region':'Basque Country','country':'ES','loc':'43.0644,-2.4898','org':'AS766 Entidad Publica Empresarial Red.es','postal':'20500','timezone':'Europe/Madrid','readme':'https://ipinfo.io/missingauth'}" } } }