Source code for yardstick.vTC.apexlake.experimental_framework.benchmarking_unit

# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

The Benchmarking Unit manages the Benchmarking of VNFs orchestrating the
initialization, execution and finalization

import json
import time
import inspect

from experimental_framework.benchmarks import benchmark_base_class as base
from experimental_framework import common
# from experimental_framework import data_manager as data
from experimental_framework import heat_template_generation as heat
from experimental_framework import deployment_unit as deploy

[docs]class BenchmarkingUnit: """ Management of the overall Benchmarking process """ def __init__(self, heat_template_name, openstack_credentials, heat_template_parameters, iterations, benchmarks): """ :param heat_template_name: (str) Name of the heat template. :param openstack_credentials: (dict) Credentials for openstack. Required fields are: 'ip_controller', 'heat_url', 'user', 'password', 'auth_uri', 'project'. :param heat_template_parameters: (dict) parameters to be given as input to the heat template. Required keys depend on the specific heat template. :param iterations: (int) number of cycles to be executed. :param benchmarks: (list[str]) List of the names of the benchmarks/test_cases to be executed in the cycle. :return: None """ # Loads vars from configuration file self.template_file_extension = common.TEMPLATE_FILE_EXTENSION self.template_dir = common.get_template_dir() self.results_directory = str(common.RESULT_DIR) + str(time.time()) # Initializes other internal variable from parameters self.template_name = heat_template_name self.iterations = iterations self.required_benchmarks = benchmarks self.template_files = [] self.benchmarks = list() self.benchmark_names = list() # self.data_manager = data.DataManager(self.results_directory) self.heat_template_parameters = heat_template_parameters self.template_files = \ heat.get_all_heat_templates(self.template_dir, self.template_file_extension) common.DEPLOYMENT_UNIT = deploy.DeploymentUnit(openstack_credentials)
[docs] def initialize(self): """ Initialize the environment in order to run the benchmarking :return: None """ for benchmark in self.required_benchmarks: benchmark_class = BenchmarkingUnit.get_benchmark_class( benchmark['name']) # Need to generate a unique name for the benchmark # (since there is the possibility to have different # instances of the same benchmark) self.benchmarks.append(benchmark_class( self.get_benchmark_name(benchmark['name']), benchmark['params'])) # for template_file_name in self.template_files: # experiment_name = BenchmarkingUnit.extract_experiment_name( # template_file_name) # self.data_manager.create_new_experiment(experiment_name) # for benchmark in self.benchmarks: # self.data_manager.add_benchmark(experiment_name, # benchmark.get_name())
[docs] def finalize(self): """ Finalizes the Benchmarking Unit Destroys all the stacks deployed by the framework and save results on csv file. :return: None """ # self.data_manager.generate_result_csv_file() common.DEPLOYMENT_UNIT.destroy_all_deployed_stacks()
[docs] def run_benchmarks(self): """ Runs all the requested benchmarks and collect the results. :return: None """'Run Benchmarking Unit') experiment = dict() result = dict() for iteration in range(0, self.iterations):'Iteration ' + str(iteration)) for template_file_name in self.template_files: experiment_name = BenchmarkingUnit.\ extract_experiment_name(template_file_name) experiment['experiment_name'] = experiment_name configuration = self.\ get_experiment_configuration(template_file_name) # self.data_manager.add_configuration(experiment_name, # configuration) for key in configuration.keys(): experiment[key] = configuration[key] # metadata = dict() # metadata['experiment_name'] = experiment_name # self.data_manager.add_metadata(experiment_name, metadata) # For each benchmark in the cycle the workload is deployed for benchmark in self.benchmarks: log_msg = 'Benchmark {} started on {}'.format( benchmark.get_name(), template_file_name ) # Initialization of Benchmark benchmark.init() log_msg = 'Template {} deployment START'.\ format(experiment_name) # Deployment of the workload deployment_success = \ common.DEPLOYMENT_UNIT.deploy_heat_template( self.template_dir + template_file_name, experiment_name, self.heat_template_parameters) if deployment_success: log_msg = 'Template {} deployment COMPLETED'.format( experiment_name) else: log_msg = 'Template {} deployment FAILED'.format( experiment_name) continue # Running the Benchmark/test case result = # self.data_manager.add_data_points(experiment_name, # benchmark.get_name(), # result) # Terminate the workload log_msg = 'Destroying deployment for experiment {}'.\ format(experiment_name) common.DEPLOYMENT_UNIT.destroy_heat_template( experiment_name) # Finalize the benchmark benchmark.finalize() log_msg = 'Benchmark {} terminated'.format( benchmark.__class__.__name__) # self.data_manager.generate_result_csv_file() experiment['benchmark'] = benchmark.get_name() for key in benchmark.get_params(): experiment[key] = benchmark.get_params()[key]'Benchmark Finished') # self.data_manager.generate_result_csv_file()'Benchmarking Unit: Experiments completed!') return result
[docs] def get_experiment_configuration(self, template_file_name): """ Reads and returns the configuration for the specific experiment (heat template) :param template_file_name: (str) Name of the file for the heat template for which it is requested the configuration :return: dict() Configuration parameters and values """ file_name = "{}{}.json".format(self.template_dir, template_file_name) with open(file_name) as json_file: configuration = json.load(json_file) return configuration
[docs] def get_benchmark_name(self, name, instance=0): """ Returns the name to be used for the benchmark/test case (TC). This is required since each benchmark/TC could be run more than once within the same cycle, with different initialization parameters. In order to distinguish between them, a unique name is generated. :param name: (str) original name of the benchmark/TC :param instance: (int) number of instance already in the queue for this type of benchmark/TC. :return: (str) name to be assigned to the benchmark/TC """ if name + "_" + str(instance) in self.benchmark_names: instance += 1 return self.get_benchmark_name(name, instance) self.benchmark_names.append(name + "_" + str(instance)) return name + "_" + str(instance)
[docs] def extract_experiment_name(template_file_name): """ Generates a unique experiment name for a given template. :param template_file_name: (str) File name of the template used during the experiment string :return: (str) Experiment Name """ strings = template_file_name.split('.') return ".".join(strings[:(len(strings)-1)])
[docs] def get_benchmark_class(complete_module_name): """ Returns the classes included in a given module. :param complete_module_name: (str) Complete name of the module as returned by get_available_test_cases. :return: Class related to the benchmark/TC present in the requested module. """ strings = complete_module_name.split('.') class_name = 'experimental_framework.benchmarks.{}'.format(strings[0]) pkg = __import__(class_name, globals(), locals(), [], -1) module = getattr(getattr(pkg, 'benchmarks'), strings[0]) members = inspect.getmembers(module) for m in members: if inspect.isclass(m[1]): class_name = m[1]("", dict()).__class__.__name__ if isinstance(m[1]("", dict()), base.BenchmarkBaseClass) and \ not class_name == 'BenchmarkBaseClass': return m[1]
[docs] def get_required_benchmarks(required_benchmarks): """ Returns instances of required test cases. :param required_benchmarks: (list() of strings) Benchmarks to be executed by the experimental framework. :return: list() of BenchmarkBaseClass """ benchmarks = list() for b in required_benchmarks: class_ = BenchmarkingUnit.get_benchmark_class(b) instance = class_("", dict()) benchmarks.append(instance) return benchmarks