diff --git a/CeTune Document.pdf b/CeTune Document.pdf index bd6b854..3dfd0cb 100644 Binary files a/CeTune Document.pdf and b/CeTune Document.pdf differ diff --git a/README.md b/README.md index b9359f6..e1c7efd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + ####Functionality Description - CeTune is a toolkit/framework to deploy, benchmark, profile and tune *Ceph cluster performance. - Aim to speed up the procedure of benchmarking *Ceph performance, and provide clear data charts of system metrics, latency breakdown data for users to analyze *Ceph performance. @@ -17,24 +18,30 @@ * * * ####Installation +- Install to head and workers: + +``` +head and workers need deploy apt-get,wget,pip proxy. +apt-get install -y python +``` + - Install to head: ``` git clone https://github.com/01org/CeTune.git -apt-get install -y python-pip pdsh unzip expect sysstat curl openjdk-7-jre haproxy python-matplotlib python-numpy python-yaml -pip install ceph-deploy argparse markdown2 -apt-get install expect +cd /CeTune/deploy/ +python controller_dependencies_install.py -#make sure head is able to autossh all worker nodes and 127.0.0.1 +# make sure head is able to autossh all worker nodes and 127.0.0.1 cd ${CeTune_PATH}/deploy/prepare-scripts; ./configure_autossh.sh ${host} ${ssh_password} ``` - Install to workers: ``` -apt-get install -y python-pip unzip sysstat curl openjdk-7-jre haproxy -apt-get install sqlite +cd /CeTune/deploy/ +python worker_dependencies_install.py ``` * * * @@ -52,13 +59,13 @@ python setup.py install cd ${CeTune_PATH}/webui/ Python webui.py -#you will see below output +# you will see below output root@client01:/CeTune/webui# python webui.py http://0.0.0.0:8080/ ``` - CeTune WebUI -![webui.png](pic/webui.png) +![webui.png](static/pic/webui.png) * * * ####Configure @@ -91,11 +98,11 @@ Assume ceph is installed on all nodes, this part is demonstrate the workflow of - Uncheck 'Benchmark' and only check 'Deploy', then click 'Execute' -![webui_deploy.png](pic/webui_deploy.png) +![webui_deploy.png](static/pic/webui_deploy.png) - WebUI will jump to 'CeTune Status' and you will about to see below console logs -![webui_deploy_detail.png](pic/webui_deploy_detail.png) +![webui_deploy_detail.png](static/pic/webui_deploy_detail.png) * * * @@ -112,11 +119,11 @@ Assume ceph is installed on all nodes, this part is demonstrate the workflow of * * * ####Check Benchmark Results -![webui_result.png](pic/webui_result.png) +![webui_result.png](static/pic/webui_result.png) -![webui_result_detail.png](pic/webui_result_detail.png) +![webui_result_detail.png](static/pic/webui_result_detail.png) -![webui_result_detail2.png](pic/webui_result_detail2.png) +![webui_result_detail2.png](static/pic/webui_result_detail2.png) * * * diff --git a/analyzer/analyzer.py b/analyzer/analyzer.py index 9a09733..ec5780c 100644 --- a/analyzer/analyzer.py +++ b/analyzer/analyzer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -* import os,sys import argparse lib_path = os.path.abspath(os.path.join('..')) @@ -13,6 +14,9 @@ import json import numpy import copy +import getpass +from multiprocessing import Process +import csv pp = pprint.PrettyPrinter(indent=4) class Analyzer: @@ -43,6 +47,8 @@ def __init__(self, dest_dir): self.cluster["osd_daemon_num"] = 0 self.cluster["perfcounter_data_type"] = self.all_conf_data.get_list("perfcounter_data_type") self.cluster["perfcounter_time_precision_level"] = self.all_conf_data.get("perfcounter_time_precision_level") + self.cluster["distributed"] = self.all_conf_data.get("distributed_data_process") + self.cluster["tmp_dir"] =self.all_conf_data.get("tmp_dir") self.result = OrderedDict() self.result["workload"] = OrderedDict() self.result["ceph"] = OrderedDict() @@ -54,8 +60,54 @@ def __init__(self, dest_dir): self.result["status"] = self.getStatus() self.result["description"] = self.getDescription() + def collect_node_ceph_version(self,dest_dir): + node_list = [] + node_list.extend(self.cluster["osds"]) + node_list.append(self.cluster["head"]) + version_list = {} + for node in node_list: + if os.path.exists(os.path.join(dest_dir,node,node+'_ceph_version.txt')): + data = open(os.path.join(dest_dir,node,node+'_ceph_version.txt'),'r') + if data: + version_list[node] = data.read().strip('\n') + else: + version_list[node] = 'None' + else: + version_list[node] = 'None' + return version_list + + def _process_remote(self,node): + common.pdsh(self.cluster['user'], [node], "cd "+ self.cluster["tmp_dir"] +";python analyzer_remote.py --path "+self.cluster["tmp_dir"] +" --name "+ node +" process_data" ) + + def _process_remote_data(self,process_file): + node_result = json.load(open(process_file,'r')) + return node_result + + def print_remote_log(self,node_log,node): + try: + get_log = "" + get_line = "wc -l "+ self.cluster["tmp_dir"] +node+"-cetune_console.log" + stdout = common.pdsh(self.cluster["user"],[node],get_line,'check_return') + h = int(stdout[0].split()[1]) + + if not node_log.has_key(node): + node_log[node] = h + get_log = "sed -n '1,"+ str(h) +"p' "+ self.cluster["tmp_dir"] +node+"-cetune_console.log" + elif h > node_log[node]: + get_log = "sed -n '"+ str(node_log[node]) +","+ str(h) +"p' "+ self.cluster["tmp_dir"] +node+"-cetune_console.log" + + if len(get_log): + log = common.pdsh(self.cluster["user"],[node],get_log,'check_return')[0] + list = log.split('\n') + for l in list: + common.printout("LOG",l) + node_log[node] = h + 1 + except Exception as e: + common.printout("WARNING","print_remote_log failed") + common.printout("WARNING",str(e)) + def process_data(self): - case_type = self.cluster["dest_dir"].split('/')[3].split('-')[2] + case_type = re.findall('\d\-\S+', self.cluster["dest_dir"])[0].split('-')[2] if case_type == "vdbench": self.result["description"] = "Description:"+ str(self.getDescription()) +" Parameters:"+ str(self.getParameters()) user = self.cluster["user"] @@ -66,31 +118,112 @@ def process_data(self): else: self.result["session_name"] = session_name[-2] - for dir_name in os.listdir(dest_dir): - if not os.path.isdir("%s/%s" % (dest_dir, dir_name)): - continue - if dir_name in self.cluster["osds"]: - self.result["ceph"][dir_name]={} - system, workload = self._process_data(dir_name) - self.result["ceph"][dir_name]=system - self.result["ceph"].update(workload) - if dir_name in self.cluster["rgw"]: - self.result["rgw"][dir_name]={} - system, workload = self._process_data(dir_name) - self.result["rgw"][dir_name]=system - self.result["rgw"].update(workload) - if dir_name in self.cluster["client"]: - self.result["client"][dir_name]={} - system, workload = self._process_data(dir_name) - self.result["client"][dir_name]=system - self.result["workload"].update(workload) - if dir_name in self.cluster["vclient"]: - params = self.result["session_name"].split('-') - self.cluster["vclient_disk"] = ["/dev/%s" % params[-1]] - self.result["vclient"][dir_name]={} - system, workload = self._process_data(dir_name) - self.result["vclient"][dir_name]=system - self.result["workload"].update(workload) + #-------------------remote start------------------------ + if self.cluster["distributed"] == "true": + self.workpath = os.path.join(self.cluster["dest_dir"],"remote_tmp") + remote_file = "../analyzer/analyzer_remote.py" + remote_file1 = "../conf/all.conf" + remote_file2 = "../conf/common.py" + remote_file3 = "../conf/config.py" + remote_file4 = "../conf/description.py" + + if not os.path.isdir(self.workpath): + os.mkdir(self.workpath) + + all_node = [] + for node in self.cluster["osds"] + self.cluster["client"]: + common.printout("LOG","note "+ node + " start analysis") + common.scp(self.cluster["user"],node,remote_file,self.cluster["tmp_dir"]) + common.scp(self.cluster["user"],node,remote_file1,self.cluster["tmp_dir"]) + common.scp(self.cluster["user"],node,remote_file2,self.cluster["tmp_dir"]) + common.scp(self.cluster["user"],node,remote_file3,self.cluster["tmp_dir"]) + common.scp(self.cluster["user"],node,remote_file4,self.cluster["tmp_dir"]) + try: + common.pdsh(self.cluster["user"],[node],"echo \"\" > " + self.cluster["tmp_dir"] +node+"-cetune_console.log") + except: + pass + + p = Process(target=self._process_remote,args=(node,)) + p.daemon = True + p.start() + all_node.append((p,node)) + + common.printout("LOG","waiting for all note finish analysis") + log_line = {} + while(1): + for proc,node in all_node: + if proc.is_alive(): + self.print_remote_log(log_line,node) + else: + common.rscp(self.cluster["user"],node,self.workpath,os.path.join(self.cluster["tmp_dir"],node+"-system.json")) + common.rscp(self.cluster["user"],node,self.workpath,os.path.join(self.cluster["tmp_dir"],node+"-workload.json")) + common.rscp(self.cluster["user"],node,self.cluster["dest_dir"].replace('raw','conf'),os.path.join(self.cluster["tmp_dir"],node+"_interrupt.csv")) + self.print_remote_log(log_line,node) + all_node.remove((proc,node)) + if not len(all_node): + break + time.sleep(1) + + common.printout("LOG","all note finish analysis") + common.printout("LOG","Merging node process.") + for dir_name in self.cluster["osds"] + self.cluster["client"]: + system_file = os.path.join(self.workpath,dir_name+"-system.json") + workload_file = os.path.join(self.workpath,dir_name+"-workload.json") + + if dir_name in self.cluster["osds"]: + self.result["ceph"][dir_name]={} + system = self._process_remote_data(system_file) + workload = self._process_remote_data(workload_file) + self.result["ceph"][dir_name]=system + self.result["ceph"].update(workload) + elif dir_name in self.cluster["rgw"]: + self.result["rgw"][dir_name]={} + system = self._process_remote_data(system_file) + workload = self._process_remote_data(workload_file) + self.result["rgw"][dir_name]=system + self.result["rgw"].update(workload) + elif dir_name in self.cluster["client"]: + self.result["client"][dir_name]={} + system = self._process_remote_data(system_file) + workload = self._process_remote_data(workload_file) + self.result["client"][dir_name]=system + self.result["workload"].update(workload) + elif dir_name in self.cluster["vclient"]: + params = self.result["session_name"].split('-') + self.cluster["vclient_disk"] = ["/dev/%s" % params[-1]] + self.result["vclient"][dir_name]={} + system = self._process_remote_data(system_file) + workload = self._process_remote_data(workload_file) + self.result["vclient"][dir_name]=system + self.result["workload"].update(workload) + #-------------------remote end-------------------------- + else: + for dir_name in os.listdir(dest_dir): + if not os.path.isdir("%s/%s" % (dest_dir, dir_name)): + continue + if dir_name in self.cluster["osds"]: + self.result["ceph"][dir_name]={} + system, workload = self._process_data(dir_name) + self.result["ceph"][dir_name]=system + self.result["ceph"].update(workload) + if dir_name in self.cluster["rgw"]: + self.result["rgw"][dir_name]={} + system, workload = self._process_data(dir_name) + self.result["rgw"][dir_name]=system + self.result["rgw"].update(workload) + if dir_name in self.cluster["client"]: + self.result["client"][dir_name]={} + system, workload = self._process_data(dir_name) + self.result["client"][dir_name]=system + self.result["workload"].update(workload) + if dir_name in self.cluster["vclient"]: + params = self.result["session_name"].split('-') + self.cluster["vclient_disk"] = ["/dev/%s" % params[-1]] + self.result["vclient"][dir_name]={} + system, workload = self._process_data(dir_name) + self.result["vclient"][dir_name]=system + self.result["workload"].update(workload) + # switch result format for visualizer # desired format @@ -113,6 +246,11 @@ def process_data(self): result = self.format_result_for_visualizer( self.result ) result = self.summary_result( result ) result["summary"]["Download"] = {"Configuration":{"URL":"" % self.result["session_name"]}} + node_ceph_version = {} + if self.collect_node_ceph_version(dest_dir): + for key,value in self.collect_node_ceph_version(dest_dir).items(): + node_ceph_version[key] = {"ceph_version":value} + result["summary"]["Node"] = node_ceph_version dest_dir = self.cluster["dest_dir_root"] common.printout("LOG","Write analyzed results into result.json") with open('%s/result.json' % dest_dir, 'w') as f: @@ -182,24 +320,28 @@ def get_execute_time(self): cf = config.Config(dest_dir+"/conf/all.conf") head = '' head = cf.get("head") - file_path = dest_dir+"raw/"+head+"/"+head+"_process_log.txt" - print file_path + file_path = os.path.join(dest_dir,"raw",head,head+"_process_log.txt") if head != '': - if os.path.exists(dest_dir+"raw/"+head+"/"): - with open(file_path, "r") as f: - lines = f.readlines() + if os.path.exists(os.path.join(dest_dir,"raw",head)): + for file_path in os.listdir(os.path.join(dest_dir,"raw",head)): + if file_path.endswith("_process_log.txt"): + with open("%s/%s" % (os.path.join(dest_dir,"raw",head),file_path), "r") as f: + lines = f.readlines() if len(lines) != 0 and lines != None: str_time = '' - str_time = lines[0].replace('CST ','') - str_time = str_time.replace('\n','') - str_time = time.strftime("%Y-%m-%d %H:%M:%S",time.strptime(str_time)) + try: + str_time = lines[0].replace('CST ','') + str_time = str_time.replace('\n','') + str_time = time.strftime("%Y-%m-%d %H:%M:%S",time.strptime(str_time)) + except: + pass return str_time else: return '' def summary_result(self, data): # generate summary - benchmark_tool = ["fio", "cosbench"] + benchmark_tool = ["fio", "cosbench", "vdbench"] data["summary"]["run_id"] = {} res = re.search('^(\d+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\d+)-(\d+)-(\w+)$',data["session_name"]) if not res: @@ -223,6 +365,7 @@ def summary_result(self, data): tmp_data["IOPS"] = 0 tmp_data["BW(MB/s)"] = 0 tmp_data["Latency(ms)"] = 0 + tmp_data["99.99%_lat(ms)"] = 0 tmp_data["SN_IOPS"] = 0 tmp_data["SN_BW(MB/s)"] = 0 tmp_data["SN_Latency(ms)"] = 0 @@ -235,6 +378,7 @@ def summary_result(self, data): write_IOPS = 0 write_BW = 0 write_Latency = 0 + max_lat = 0 for engine_candidate in data["workload"].keys(): if engine_candidate in benchmark_tool: engine = engine_candidate @@ -246,6 +390,7 @@ def summary_result(self, data): write_IOPS += float(node_data["write_iops"]) write_BW += float(node_data["write_bw"]) write_Latency += float(node_data["write_lat"]) + max_lat += float(node_data["99.99%_lat"]) if tmp_data["Op_Type"] in ["randread", "seqread", "read"]: tmp_data["IOPS"] = "%.3f" % read_IOPS tmp_data["BW(MB/s)"] = "%.3f" % read_BW @@ -261,6 +406,8 @@ def summary_result(self, data): tmp_data["BW(MB/s)"] = "%.3f, %.3f" % (read_BW, write_BW) if rbd_count > 0: tmp_data["Latency(ms)"] = "%.3f, %.3f" % ((read_Latency/rbd_count), (write_Latency/rbd_count)) + if rbd_count > 0: + tmp_data["99.99%_lat(ms)"] = "%.3f" % (max_lat/rbd_count) except: pass read_SN_IOPS = 0 @@ -269,7 +416,7 @@ def summary_result(self, data): write_SN_IOPS = 0 write_SN_BW = 0 write_SN_Latency = 0 - diskformat = common.parse_disk_format( self.cluster['diskformat'] ) + diskformat = common.parse_disk_format( self.cluster['diskformat'] ) if len(diskformat): typename = diskformat[0] else: @@ -326,6 +473,8 @@ def _process_data(self, node_name): workload_result.update(self.process_cosbench_data("%s/%s/%s" %(dest_dir, node_name, dir_name), dir_name)) if '_sar.txt' in dir_name: result.update(self.process_sar_data("%s/%s/%s" % (dest_dir, node_name, dir_name))) + if 'totals.html' in dir_name: + workload_result.update(self.process_vdbench_data("%s/%s/%s" % (dest_dir, node_name, dir_name), "%s_%s" % (node_name, dir_name))) if '_fio.txt' in dir_name: workload_result.update(self.process_fio_data("%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name)) if '_fio_iops.1.log' in dir_name or '_fio_bw.1.log' in dir_name or '_fio_lat.1.log' in dir_name: @@ -345,8 +494,8 @@ def _process_data(self, node_name): result.update(res) if '_interrupts_end.txt' in dir_name: if os.path.exists("%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start'))): - interrupt_start = "%s/%s/%s" % (dest_dir, node_name, dir_name) - interrupt_end = "%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start')) + interrupt_end = "%s/%s/%s" % (dest_dir, node_name, dir_name) + interrupt_start = "%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start')) self.interrupt_diff(dest_dir,node_name,interrupt_start,interrupt_end) if '_process_log.txt' in dir_name: res = self.process_log_data( "%s/%s/%s" % (dest_dir, node_name, dir_name) ) @@ -372,8 +521,9 @@ def process_smartinfo_data(self, path): def interrupt_diff(self,dest_dir,node_name,s_path,e_path): s_p = s_path e_p = e_path - result_name = node_name+'_interrupt.txt' - result_path = os.path.join(dest_dir.replace('raw','conf'),result_name) + result_name = node_name+'_interrupt.csv' + result_path_node = os.path.join(dest_dir,node_name,result_name) + result_path_conf = os.path.join(dest_dir.replace('raw','conf'),result_name) s_l = [] e_l = [] diff_list = [] @@ -398,25 +548,42 @@ def interrupt_diff(self,dest_dir,node_name,s_path,e_path): lines = [] for j in range(len(s_l[i])): if s_l[i][j].isdigit() and e_l[i][j].isdigit(): - diff_value = int(e_l[i][j]) - int(s_l[i][j]) lines.append(int(e_l[i][j]) - int(s_l[i][j])) else: lines.append(e_l[i][j]) diff_list.append(lines) - if os.path.exists(result_path): - os.remove(result_path) - output = open(result_path,'w+') - for line in diff_list: - line_str = '' - for col in range(len(line)): - if col != len(line)-1: - line_str += str(line[col])+' ' - else: - line_str += str(line[col]) - output.writelines(line_str) - output.close() + ##write interrupt to node and conf + common.printout("LOG","write interrput to node and conf.") + if os.path.exists(result_path_node): + os.remove(result_path_node) + if os.path.exists(result_path_conf): + os.remove(result_path_conf) + output_node = file(result_path_node,'wb') + output_conf = file(result_path_conf,'wb') + interrupt_csv_node = csv.writer(output_node) + interrupt_csv_conf = csv.writer(output_conf) + if len(diff_list) != 0: + diff_list[0][0] = "" + interrupt_csv_node.writerow(diff_list[0]) + interrupt_csv_conf.writerow(diff_list[0]) + del diff_list[0] + new_diff_list = self.delete_colon(diff_list) + for i in new_diff_list: + interrupt_csv_node.writerows([i]) + interrupt_csv_conf.writerows([i]) + output_node.close() + output_conf.close() + else: + common.printout("WARNING","no interrupt.") else: - print 'ERROR: interrupt_start lines and interrupt_end lines are diffrent ! can not calculate diffrent value!' + common.printout("ERROR",'interrupt_start lines and interrupt_end lines are different ! can not calculate different value!') + + def delete_colon(self,data_list): + self.d_list = data_list + for i in range(len(self.d_list)): + self.d_list[i][0] = self.d_list[i][0].replace(":","") + self.d_list[i][-1] = self.d_list[i][-1].strip("\n") + return self.d_list def check_interrupt(self,s_inter,e_inter): result = "True" @@ -517,7 +684,7 @@ def getParameters(self): dest_dir = self.cluster["dest_conf_dir"] ps = "" try: - with open("%s/parameters.cfg" % dest_dir.replace("conf","raw"), 'r') as f: + with open("%s/vdbench_params.txt" % dest_dir.replace("raw","conf"), 'r') as f: ps = f.read() except: pass @@ -547,18 +714,33 @@ def process_fiolog_data(self, path, result): time_shift = 1000 with open( path, "r" ) as f: cur_sec = -1 - for line in f.readlines(): - data = line.split(",") - timestamp_sec = int(data[0])/time_shift - value = int(data[1]) - while ( timestamp_sec > (cur_sec + 1) ): - res.append( 0 ) - cur_sec += 1 - if (cur_sec + 1) == timestamp_sec: - res.append( value ) - cur_sec += 1 - elif cur_sec == timestamp_sec: - res[-1] = (res[-1] + value)/2 + self.tmp_res = [] + if 'iops' in path: + self.iops_value = 0 + for line in f.readlines(): + data = line.split(",") + value = int(data[1]) + timestamp_sec = int(data[0])/time_shift + if timestamp_sec > cur_sec: + if cur_sec >= 0: + self.tmp_res.append( self.iops_value ) + self.iops_value = 0 + cur_sec = timestamp_sec + self.iops_value += value + if len(self.tmp_res) != 0: + res.extend(self.tmp_res) + else: + for line in f.readlines(): + data = line.split(",") + timestamp_sec = int(data[0])/time_shift + value = int(data[1]) + if timestamp_sec > cur_sec: + if cur_sec >= 0: + res.append(numpy.mean(self.tmp_res)) + cur_sec = timestamp_sec + self.tmp_res.append( value ) + if len(self.tmp_res) != 0: + res.append(numpy.mean(self.tmp_res)) return result @@ -590,7 +772,8 @@ def process_iostat_data(self, node, path): tmp_dev_name = osd_journal[i].split('/')[2] if 'nvme' in tmp_dev_name: tmp_dev_name = common.parse_nvme( tmp_dev_name ) - disk_list.append( tmp_dev_name ) + if tmp_dev_name not in disk_list: + disk_list.append( tmp_dev_name ) dict_diskformat[output_list[i]]=disk_list elif node in self.cluster["vclient"]: vdisk_list = [] @@ -602,7 +785,7 @@ def process_iostat_data(self, node, path): for output in output_list: if output != "vdisk": disk_list = " ".join(dict_diskformat[output]) - disk_num = len(dict_diskformat[output]) + disk_num = len(list(set(dict_diskformat[output]))) else: disk_list = " ".join(vdisk_list) disk_num = len(vdisk_list) @@ -611,9 +794,50 @@ def process_iostat_data(self, node, path): result[output]["disk_num"] = disk_num return result + def process_vdbench_data(self, path, dirname): + result = {} + vdbench_data = {} + runtime = int(common.bash("grep -o 'elapsed=[0-9]\+' "+path+" | cut -d = -f 2")) + stdout, stderr = common.bash("grep 'avg_2-' "+path, True) + vdbench_data = stdout.split() + output_vdbench_data = OrderedDict() + output_vdbench_data['read_lat'] = vdbench_data[8] + output_vdbench_data["read_iops"] = vdbench_data[7] + output_vdbench_data["read_bw"] = vdbench_data[11] + output_vdbench_data['read_runtime'] = runtime + output_vdbench_data['write_lat'] = vdbench_data[10] + output_vdbench_data["write_iops"] = vdbench_data[9] + output_vdbench_data["write_bw"] = vdbench_data[12] + output_vdbench_data['write_runtime'] = runtime + output_vdbench_data['lat_unit'] = 'msec' + output_vdbench_data['runtime_unit'] = 'sec' + output_vdbench_data['bw_unit'] = 'MB/s' + result[dirname] = {} + result[dirname]["vdbench"] = output_vdbench_data + return result + + def get_lat_persent_dict(self,fio_str): + lat_percent_dict = {} + tmp_list = fio_str.split(',') + for i in tmp_list: + li = i.split('=') + while '' in li:li.remove('') + if len(li) == 2 and li[1] != '': + key = re.findall('.*?th',li[0].strip('\n').strip('| ').strip(' ').replace(' ',''),re.S) + value = re.match(r'\[(.*?)\]',li[1].strip('\n').strip(' ').replace(' ','')).groups() + if len(key) != 0 and len(value) != 0: + lat_percent_dict[key[0]] = value[0] + return lat_percent_dict + def process_fio_data(self, path, dirname): result = {} stdout, stderr = common.bash("grep \" *io=.*bw=.*iops=.*runt=.*\|^ *lat.*min=.*max=.*avg=.*stdev=.*\" "+path, True) + stdout1, stderr1 = common.bash("grep \" *1.00th.*],\| *30.00th.*],\| *70.00th.*],\| *99.00th.*],\| *99.99th.*]\" "+path, True) + stdout2, stderr2 = common.bash("grep \" *clat percentiles\" "+path, True) + lat_per_dict = {} + if stdout1 != '': + lat_per_dict = self.get_lat_persent_dict(stdout1) + fio_data_rw = {} fio_data_rw["read"] = {} fio_data_rw["write"] = {} @@ -643,6 +867,16 @@ def process_fio_data(self, path, dirname): output_fio_data['write_iops'] = 0 output_fio_data['write_bw'] = 0 output_fio_data['write_runtime'] = 0 + if len(lat_per_dict) != 0: + if '99.99th' in lat_per_dict.keys(): + #output_fio_data['99.99%_lat'] = lat_per_dict['99.99th'] + lat_persent_unit = re.findall(r"(?<=[\(])[^\)]+(?=[\)])", stdout2.strip('\n').strip(' ').replace(' ','')) + if len(lat_persent_unit) != 0: + output_fio_data['99.99%_lat'] = float(common.time_to_sec("%s%s" % (lat_per_dict['99.99th'], lat_persent_unit[0]),'msec')) + else: + output_fio_data['99.99%_lat'] = 'null' + else: + output_fio_data['99.99%_lat'] = 'null' output_fio_data['lat_unit'] = 'msec' output_fio_data['runtime_unit'] = 'sec' output_fio_data['bw_unit'] = 'MB/s' diff --git a/analyzer/analyzer_remote.py b/analyzer/analyzer_remote.py new file mode 100644 index 0000000..5b5d001 --- /dev/null +++ b/analyzer/analyzer_remote.py @@ -0,0 +1,892 @@ +# -*- coding: utf-8 -* +import os,sys +import argparse +lib_path = os.path.abspath(os.path.join('..')) +sys.path.append(lib_path) +import common as cn +import os, sys +import time +import pprint +import re +import yaml +from collections import OrderedDict +import json +import numpy +import copy +import config +from multiprocessing import Process +import csv + +pp = pprint.PrettyPrinter(indent=4) +class Analyzer: + def __init__(self, dest_dir,name): + self.common = cn + self.common.cetune_log_file = name+"-cetune_process.log" + self.common.cetune_error_file = name+"-cetune_error.log" + self.common.cetune_console_file= name+"-cetune_console.log" + + self.dest_dir = dest_dir + self.cluster = {} + self.cluster["dest_dir"] = dest_dir + self.cluster["dest_conf_dir"] = dest_dir + self.cluster["dest_dir_root"] = dest_dir + self.all_conf_data = config.Config("all.conf") + self.cluster["user"] = self.all_conf_data.get("user") + self.cluster["head"] = self.all_conf_data.get("head") + self.cluster["diskformat"] = self.all_conf_data.get("disk_format", dotry=True) + self.cluster["client"] = self.all_conf_data.get_list("list_client") + self.cluster["osds"] = self.all_conf_data.get_list("list_server") + self.cluster["mons"] = self.all_conf_data.get_list("list_mon") + self.cluster["rgw"] = self.all_conf_data.get_list("rgw_server") + self.cluster["vclient"] = self.all_conf_data.get_list("list_vclient") + self.cluster["head"] = self.all_conf_data.get("head") + self.cluster["user"] = self.all_conf_data.get("user") + self.cluster["monitor_interval"] = self.all_conf_data.get("monitoring_interval") + self.cluster["osd_daemon_num"] = 0 + self.cluster["perfcounter_data_type"] = self.all_conf_data.get_list("perfcounter_data_type") + self.cluster["perfcounter_time_precision_level"] = self.all_conf_data.get("perfcounter_time_precision_level") + self.cluster["distributed"] = self.all_conf_data.get("distributed_data_process") + self.cluster["tmp_dir"] =self.all_conf_data.get("tmp_dir") + self.result = OrderedDict() + self.result["workload"] = OrderedDict() + self.result["ceph"] = OrderedDict() + self.result["rgw"] = OrderedDict() + self.result["client"] = OrderedDict() + self.result["vclient"] = OrderedDict() + self.get_validate_runtime() + self.result["runtime"] = int(float(self.validate_time)) + self.result["status"] = self.getStatus() + self.result["description"] = self.getDescription() + + self.whoami = name + + + def collect_node_ceph_version(self,dest_dir): + node_list = [] + node_list.extend(self.cluster["osds"]) + node_list.append(self.cluster["head"]) + version_list = {} + for node in node_list: + if os.path.exists(os.path.join(dest_dir,node,node+'_ceph_version.txt')): + data = open(os.path.join(dest_dir,node,node+'_ceph_version.txt'),'r') + if data: + version_list[node] = data.read().strip('\n') + else: + version_list[node] = 'None' + else: + version_list[node] = 'None' + return version_list + + def test_write_json(self,data,file): + json.dump(data,open(file,'w')) + + + def process_data(self): + process_list = [] + user = self.cluster["user"] + dest_dir = self.cluster["dest_dir"] + session_name = self.cluster["dest_dir_root"].split('/') + if session_name[-1] != '': + self.result["session_name"] = session_name[-1] + else: + self.result["session_name"] = session_name[-2] + + if self.whoami in self.cluster["osds"]: + self.result["ceph"][self.whoami]={} + p = Process(target=self._process_data,args=()) + process_list.append(p) + if self.whoami in self.cluster["rgw"]: + self.result["rgw"][self.whoami]={} + p = Process(target=self._process_data,args=()) + process_list.append(p) + if self.whoami in self.cluster["client"]: + self.result["client"][self.whoami]={} + p = Process(target=self._process_data,args=()) + process_list.append(p) + if self.whoami in self.cluster["vclient"]: + params = self.result["session_name"].split('-') + self.cluster["vclient_disk"] = ["/dev/%s" % params[-1]] + self.result["vclient"][self.whoami]={} + p = Process(target=self._process_data,args=()) + process_list.append(p) + + for poc in process_list: + poc.daemon = True + poc.start() + poc.join() + + + return + + # switch result format for visualizer + # desired format + ''' + result = { + tab1: { + table1: { + row1: { + column1: [value], column2: [value] , ... + } + } + } + }, + tab2: {} + tab3: {} + ... + tabn: {} + } + ''' + result = self.format_result_for_visualizer( self.result ) + result = self.summary_result( result ) + result["summary"]["Download"] = {"Configuration":{"URL":"" % self.result["session_name"]}} + node_ceph_version = {} + if self.collect_node_ceph_version(dest_dir): + for key,value in self.collect_node_ceph_version(dest_dir).items(): + node_ceph_version[key] = {"ceph_version":value} + result["summary"]["Node"] = node_ceph_version + dest_dir = self.cluster["dest_dir_root"] + self.common.printout("LOG","Write analyzed results into result.json") + with open('%s/result.json' % dest_dir, 'w') as f: + json.dump(result, f, indent=4) + view = visualizer.Visualizer(result, dest_dir) + output = view.generate_summary_page() + + def format_result_for_visualizer(self, data): + output_sort = OrderedDict() + monitor_interval = int(self.cluster["monitor_interval"]) + output_sort["summary"] = OrderedDict() + res = re.search('^(\d+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\d+)-(\d+)-(\w+)$',data["session_name"]) + if not res: + return output_sort + rampup = int(res.group(8)) + runtime = int(res.group(9)) + diskformat = self.common.parse_disk_format( self.cluster['diskformat'] ) + phase_name_map_for_disk = {} + for typename in diskformat: + phase_name_map_for_disk[typename] = "iostat" + phase_name_map = {"cpu": "sar", "memory": "sar", "nic": "sar", "vdisk": "iostat" } + phase_name_map.update( phase_name_map_for_disk ) + + for node_type in data.keys(): + if not isinstance(data[node_type], dict): + output_sort[node_type] = data[node_type] + continue + if data[node_type] == {}: + continue + output = {} + output_sort[node_type] = OrderedDict() + for node in sorted(data[node_type].keys()): + for field_type in sorted(data[node_type][node].keys()): + if field_type == "phase": + continue + if field_type not in output: + output[field_type] = OrderedDict() + if "phase" in data[node_type][node].keys() and field_type in phase_name_map.keys(): + try: + start = int(data[node_type][node]["phase"][phase_name_map[field_type]]["benchmark_start"]) + end = int(data[node_type][node]["phase"][phase_name_map[field_type]]["benchmark_stop"]) + benchmark_active_time = end - start + if benchmark_active_time > (rampup + runtime) or end <= 0: + runtime_end = start + rampup + runtime + else: + runtime_end = end + runtime_start = start + rampup + output[field_type][node] = OrderedDict() + runtime_start = runtime_start / monitor_interval + runtime_end = runtime_end / monitor_interval + + for colume_name, colume_data in data[node_type][node][field_type].items(): + if isinstance(colume_data, list): + colume_data = colume_data[runtime_start:runtime_end] + output[field_type][node][colume_name] = colume_data + except: + output[field_type][node] = data[node_type][node][field_type] + else: + output[field_type][node] = data[node_type][node][field_type] + for key in sorted(output.keys()): + output_sort[node_type][key] = copy.deepcopy( output[key] ) + + return output_sort + + def get_execute_time(self): + dest_dir = self.dest_dir + cf = config.Config(dest_dir+"/conf/all.conf") + head = '' + head = cf.get("head") + file_path = os.path.join(dest_dir,"raw",head,head+"_process_log.txt") + if head != '': + if os.path.exists(os.path.join(dest_dir,"raw",head)): + for file_path in os.listdir(os.path.join(dest_dir,"raw",head)): + if file_path.endswith("_process_log.txt"): + with open("%s/%s" % (os.path.join(dest_dir,"raw",head),file_path), "r") as f: + lines = f.readlines() + if len(lines) != 0 and lines != None: + str_time = '' + try: + str_time = lines[0].replace('CST ','') + str_time = str_time.replace('\n','') + str_time = time.strftime("%Y-%m-%d %H:%M:%S",time.strptime(str_time)) + except: + pass + return str_time + else: + return '' + + def summary_result(self, data): + # generate summary + benchmark_tool = ["fio", "cosbench", "vdbench"] + data["summary"]["run_id"] = {} + res = re.search('^(\d+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\w+)-(\d+)-(\d+)-(\w+)$',data["session_name"]) + if not res: + self.common.printout("ERROR", "Unable to get result infomation") + return data + data["summary"]["run_id"][res.group(1)] = OrderedDict() + tmp_data = data["summary"]["run_id"][res.group(1)] + tmp_data["Timestamp"] = self.get_execute_time() + tmp_data["Status"] = data["status"] + tmp_data["Description"] = data["description"] + tmp_data["Op_size"] = res.group(5) + tmp_data["Op_Type"] = res.group(4) + tmp_data["QD"] = res.group(6) + tmp_data["Driver"] = res.group(3) + tmp_data["SN_Number"] = 0 + tmp_data["CN_Number"] = 0 + tmp_data["Worker"] = res.group(2) + if data["runtime"] == 0: + data["runtime"] = int(res.group(9)) + tmp_data["Runtime"] = "%d" % (data["runtime"]) + tmp_data["IOPS"] = 0 + tmp_data["BW(MB/s)"] = 0 + tmp_data["Latency(ms)"] = 0 + tmp_data["SN_IOPS"] = 0 + tmp_data["SN_BW(MB/s)"] = 0 + tmp_data["SN_Latency(ms)"] = 0 + rbd_count = 0 + osd_node_count = 0 + try: + read_IOPS = 0 + read_BW = 0 + read_Latency = 0 + write_IOPS = 0 + write_BW = 0 + write_Latency = 0 + for engine_candidate in data["workload"].keys(): + if engine_candidate in benchmark_tool: + engine = engine_candidate + for node, node_data in data["workload"][engine].items(): + rbd_count += 1 + read_IOPS += float(node_data["read_iops"]) + read_BW += float(node_data["read_bw"]) + read_Latency += float(node_data["read_lat"]) + write_IOPS += float(node_data["write_iops"]) + write_BW += float(node_data["write_bw"]) + write_Latency += float(node_data["write_lat"]) + if tmp_data["Op_Type"] in ["randread", "seqread", "read"]: + tmp_data["IOPS"] = "%.3f" % read_IOPS + tmp_data["BW(MB/s)"] = "%.3f" % read_BW + if rbd_count > 0: + tmp_data["Latency(ms)"] = "%.3f" % (read_Latency/rbd_count) + elif tmp_data["Op_Type"] in ["randwrite", "seqwrite", "write"]: + tmp_data["IOPS"] = "%.3f" % write_IOPS + tmp_data["BW(MB/s)"] = "%.3f" % write_BW + if rbd_count > 0: + tmp_data["Latency(ms)"] = "%.3f" % (write_Latency/rbd_count) + elif tmp_data["Op_Type"] in ["randrw", "rw", "readwrite"]: + tmp_data["IOPS"] = "%.3f, %.3f" % (read_IOPS, write_IOPS) + tmp_data["BW(MB/s)"] = "%.3f, %.3f" % (read_BW, write_BW) + if rbd_count > 0: + tmp_data["Latency(ms)"] = "%.3f, %.3f" % ((read_Latency/rbd_count), (write_Latency/rbd_count)) + except: + pass + read_SN_IOPS = 0 + read_SN_BW = 0 + read_SN_Latency = 0 + write_SN_IOPS = 0 + write_SN_BW = 0 + write_SN_Latency = 0 + diskformat = self.common.parse_disk_format( self.cluster['diskformat'] ) + if len(diskformat): + typename = diskformat[0] + else: + typename = "osd" + for node, node_data in data["ceph"][typename].items(): + osd_node_count += 1 + read_SN_IOPS += numpy.mean(node_data["r/s"])*int(node_data["disk_num"]) + read_SN_BW += numpy.mean(node_data["rMB/s"])*int(node_data["disk_num"]) + lat_name = "r_await" + if lat_name not in node_data: + lat_name = "await" + read_SN_Latency += numpy.mean(node_data[lat_name]) + write_SN_IOPS += numpy.mean(node_data["w/s"])*int(node_data["disk_num"]) + write_SN_BW += numpy.mean(node_data["wMB/s"])*int(node_data["disk_num"]) + lat_name = "w_await" + if lat_name not in node_data: + lat_name = "await" + write_SN_Latency += numpy.mean(node_data[lat_name]) + + if tmp_data["Op_Type"] in ["randread", "seqread", "read"]: + tmp_data["SN_IOPS"] = "%.3f" % read_SN_IOPS + tmp_data["SN_BW(MB/s)"] = "%.3f" % read_SN_BW + if osd_node_count > 0: + tmp_data["SN_Latency(ms)"] = "%.3f" % (read_SN_Latency/osd_node_count) + elif tmp_data["Op_Type"] in ["randwrite", "seqwrite", "write"]: + tmp_data["SN_IOPS"] = "%.3f" % write_SN_IOPS + tmp_data["SN_BW(MB/s)"] = "%.3f" % write_SN_BW + if osd_node_count > 0: + tmp_data["SN_Latency(ms)"] = "%.3f" % (write_SN_Latency/osd_node_count) + elif tmp_data["Op_Type"] in ["randrw", "readwrite", "rw"]: + tmp_data["SN_IOPS"] = "%.3f, %.3f" % (read_SN_IOPS, write_SN_IOPS) + tmp_data["SN_BW(MB/s)"] = "%.3f, %.3f" % (read_SN_BW, write_SN_BW) + if osd_node_count > 0: + tmp_data["SN_Latency(ms)"] = "%.3f, %.3f" % (read_SN_Latency/osd_node_count, write_SN_Latency/osd_node_count) + + tmp_data["SN_Number"] = osd_node_count + try: + tmp_data["CN_Number"] = len(data["client"]["cpu"]) + except: + tmp_data["CN_Number"] = 0 + return data + + def _process_data(self): + result = {} + fio_log_res = {} + workload_result = {} + dest_dir = self.cluster["dest_dir"] + self.common.printout("LOG","dest_dir:%s"%dest_dir) + for dir_name in os.listdir(dest_dir): + if 'smartinfo.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + res = self.process_smartinfo_data( "%s/%s" % (dest_dir, dir_name)) + result.update(res) + if 'cosbench' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + workload_result.update(self.process_cosbench_data("%s/%s" %(dest_dir, dir_name), dir_name)) + if '_sar.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + result.update(self.process_sar_data("%s/%s" % (dest_dir, dir_name))) + if 'totals.html' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + workload_result.update(self.process_vdbench_data("%s/%s" % (dest_dir, dir_name), "%s_%s" % (self.whoami, dir_name))) + if '_fio.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + workload_result.update(self.process_fio_data("%s/%s" % (dest_dir, dir_name), dir_name)) + if '_fio_iops.1.log' in dir_name or '_fio_bw.1.log' in dir_name or '_fio_lat.1.log' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + if "_fio_iops.1.log" in dir_name: + volume = dir_name.replace("_fio_iops.1.log", "") + if "_fio_bw.1.log" in dir_name: + volume = dir_name.replace("_fio_bw.1.log", "") + if "_fio_lat.1.log" in dir_name: + volume = dir_name.replace("_fio_lat.1.log", "") + if volume not in fio_log_res: + fio_log_res[volume] = {} + fio_log_res[volume]["fio_log"] = {} + fio_log_res[volume]["fio_log"] = self.process_fiolog_data("%s/%s" % (dest_dir, dir_name), fio_log_res[volume]["fio_log"] ) + workload_result.update(fio_log_res) + if '_iostat.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + res = self.process_iostat_data( self.whoami, "%s/%s" % (dest_dir, dir_name)) + result.update(res) + if '_interrupts_end.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + if os.path.exists("%s/%s" % (dest_dir, dir_name.replace('end','start'))): + interrupt_end = "%s/%s" % (dest_dir, dir_name) + interrupt_start = "%s/%s" % (dest_dir, dir_name.replace('end','start')) + self.interrupt_diff(dest_dir,self.whoami,interrupt_start,interrupt_end) + if '_process_log.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + res = self.process_log_data( "%s/%s" % (dest_dir, dir_name) ) + result.update(res) + if '.asok.txt' in dir_name: + self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) + try: + res = self.process_perfcounter_data("%s/%s" % (dest_dir, dir_name)) + for key, value in res.items(): + if dir_name not in workload_result: + workload_result[dir_name] = OrderedDict() + workload_result[dir_name][key] = value + except: + pass + self.test_write_json(result,self.whoami+"-system.json") + self.test_write_json(workload_result,self.whoami+"-workload.json") + return [result, workload_result] + + def process_smartinfo_data(self, path): + output = {} + with open(path, 'r') as f: + tmp = f.read() + output.update(json.loads(tmp, object_pairs_hook=OrderedDict)) + return output + + def interrupt_diff(self,dest_dir,node_name,s_path,e_path): + s_p = s_path + e_p = e_path + result_name = node_name+'_interrupt.csv' + result_path_node = os.path.join(dest_dir,result_name) + s_l = [] + e_l = [] + diff_list = [] + with open(s_p, 'r') as f: + s = f.readlines() + with open(e_p, 'r') as f: + e = f.readlines() + for i in s: + tmp = [] + tmp = i.split(' ') + while '' in tmp: + tmp.remove('') + s_l.append(tmp) + for i in e: + tmp = [] + tmp = i.split(' ') + while '' in tmp: + tmp.remove('') + e_l.append(tmp) + if self.check_interrupt(s_l,e_l): + for i in range(len(s_l)): + lines = [] + for j in range(len(s_l[i])): + if s_l[i][j].isdigit() and e_l[i][j].isdigit(): + lines.append(int(e_l[i][j]) - int(s_l[i][j])) + else: + lines.append(e_l[i][j]) + diff_list.append(lines) + ##write interrupt to node and conf + self.common.printout("LOG","write interrput to node and conf.") + if os.path.exists(result_path_node): + os.remove(result_path_node) + output_node = file(result_path_node,'wb') + interrupt_csv_node = csv.writer(output_node) + if len(diff_list) != 0: + diff_list[0][0] = "" + interrupt_csv_node.writerow(diff_list[0]) + del diff_list[0] + new_diff_list = self.delete_colon(diff_list) + for i in new_diff_list: + interrupt_csv_node.writerows([i]) + output_node.close() + else: + self.common.printout("WARNING","no interrupt.") + else: + self.common.printout("ERROR",'interrupt_start lines and interrupt_end lines are different ! can not calculate different value!') + + def delete_colon(self,data_list): + self.d_list = data_list + for i in range(len(self.d_list)): + self.d_list[i][0] = self.d_list[i][0].replace(":","") + self.d_list[i][-1] = self.d_list[i][-1].strip("\n") + return self.d_list + + def check_interrupt(self,s_inter,e_inter): + result = "True" + if len(s_inter)!=len(e_inter): + result = "False" + else: + for i in range(len(s_inter)): + if len(s_inter[i])!=len(e_inter[i]): + result = "False" + return result + + def process_log_data(self, path): + result = {} + result["phase"] = {} + with open( path, 'r') as f: + lines = f.readlines() + + benchmark_tool = ["fio", "cosbench"] + tmp = {} + benchmark = {} + + for line in lines: + try: + time, tool, status = line.split() + except: + continue + if tool not in tmp: + tmp[tool] = {} + if tool in benchmark_tool: + benchmark[status] = time + else: + tmp[tool][status] = time + + for tool in tmp: + result["phase"][tool] = {} + result["phase"][tool]["start"] = 0 + try: + result["phase"][tool]["stop"] = int(tmp[tool]["stop"]) - int(tmp[tool]["start"]) + except: + result["phase"][tool]["stop"] = None + try: + result["phase"][tool]["benchmark_start"] = int(benchmark["start"]) - int(tmp[tool]["start"]) + if result["phase"][tool]["benchmark_start"] < 0: + result["phase"][tool]["benchmark_start"] = 0 + except: + result["phase"][tool]["benchmark_start"] = None + try: + result["phase"][tool]["benchmark_stop"] = int(benchmark["stop"]) - int(tmp[tool]["start"]) + if result["phase"][tool]["benchmark_stop"] < 0: + result["phase"][tool]["benchmark_stop"] = 0 + except: + result["phase"][tool]["benchmark_stop"] = None + return result + + def process_cosbench_data(self, path, dirname): + result = {} + result["cosbench"] = OrderedDict() + result["cosbench"]["cosbench"] = OrderedDict([("read_lat",0), ("read_bw",0), ("read_iops",0), ("write_lat",0), ("write_bw",0), ("write_iops",0), ("lat_unit",'msec'), ('runtime_unit','sec'), ('bw_unit','MB/s')]) + tmp = result + keys = self.common.bash("head -n 1 %s/%s.csv" %(path, dirname)) + keys = keys.split(',') + values = self.common.bash('tail -n 1 %s/%s.csv' %(path, dirname) ) + values = values.split(',') + size = len(keys) + for i in range(size): + tmp[keys[i]] = {} + tmp[keys[i]]["detail"] = {} + tmp[keys[i]]["detail"]["value"] = values[i] + tmp = result["cosbench"]["cosbench"] + io_pattern = result["Op-Type"]["detail"]["value"] + tmp["%s_lat" % io_pattern] = result["Avg-ResTime"]["detail"]["value"] + tmp["%s_bw" % io_pattern] = self.common.size_to_Kbytes('%s%s' % (result["Bandwidth"]["detail"]["value"], 'B'), 'MB') + tmp["%s_iops" % io_pattern] = result["Throughput"]["detail"]["value"] + return result + + def get_validate_runtime(self): + self.validate_time = 0 + dest_dir = self.cluster["dest_dir"] + stdout = self.common.bash('grep " runt=.*" -r %s' % (dest_dir)) + fio_runtime_list = re.findall('runt=\s*(\d+\wsec)', stdout) + for fio_runtime in fio_runtime_list: + validate_time = self.common.time_to_sec(fio_runtime, 'sec') + if validate_time < self.validate_time or self.validate_time == 0: + self.validate_time = validate_time + + def getStatus(self): + self.validate_time = 0 + dest_dir = self.cluster["dest_conf_dir"] + status = "Unknown" + try: + with open("%s/status" % dest_dir, 'r') as f: + status = f.readline() + except: + pass + return status + + def getParameters(self): + dest_dir = self.cluster["dest_conf_dir"] + ps = "" + try: + with open("%s/vdbench_params.txt" % dest_dir.replace("raw","conf"), 'r') as f: + ps = f.read() + except: + pass + return ps + + def getDescription(self): + dest_dir = self.cluster["dest_conf_dir"] + desc = "" + try: + with open("%s/description" % dest_dir, 'r') as f: + desc = f.readline() + except: + pass + return desc + + def process_fiolog_data(self, path, result): + if "fio_iops" in path: + result["iops"] = [] + res = result["iops"] + if "fio_bw" in path: + result["bw"] = [] + res = result["bw"] + if "fio_lat" in path: + result["lat"] = [] + res = result["lat"] + + time_shift = 1000 + with open( path, "r" ) as f: + cur_sec = -1 + self.tmp_res = [] + if 'iops' in path: + self.iops_value = 0 + for line in f.readlines(): + data = line.split(",") + value = int(data[1]) + timestamp_sec = int(data[0])/time_shift + if timestamp_sec > cur_sec: + if cur_sec >= 0: + self.tmp_res.append( self.iops_value ) + self.iops_value = 0 + cur_sec = timestamp_sec + self.iops_value += value + if len(self.tmp_res) != 0: + res.extend(self.tmp_res) + else: + for line in f.readlines(): + data = line.split(",") + timestamp_sec = int(data[0])/time_shift + value = int(data[1]) + if timestamp_sec > cur_sec: + if cur_sec >= 0: + res.append(numpy.mean(self.tmp_res)) + cur_sec = timestamp_sec + self.tmp_res.append( value ) + if len(self.tmp_res) != 0: + res.append(numpy.mean(self.tmp_res)) + return result + + + def process_sar_data(self, path): + result = {} + #1. cpu + stdout = self.common.bash("grep ' *CPU *%' -m 1 "+path+" | awk -F\"CPU\" '{print $2}'; cat "+path+" | grep ' *CPU *%' -A 1 | awk '{flag=0;if(NF<=3)next;for(i=1;i<=NF;i ++){if(flag==1){printf $i\"\"FS}if($i==\"all\")flag=1};if(flag==1)print \"\"}'") + result["cpu"] = self.common.convert_table_to_2Dlist(stdout) + + #2. memory + stdout = self.common.bash("grep 'kbmemfree' -m 1 "+path+" | awk -Fkbmemfree '{printf \"kbmenfree \";print $2}'; grep \"kbmemfree\" -A 1 "+path+" | awk 'BEGIN{find=0;} {for(i=1;i<=NF;i++){if($i==\"kbmemfree\"){find=i;next;}}if(find!=0){for(j=find;j<=NF;j++)printf $j\"\"FS;find=0;print \"\"}}'") + result["memory"] = self.common.convert_table_to_2Dlist(stdout) + + #3. nic + stdout = self.common.bash("grep 'IFACE' -m 1 "+path+" | awk -FIFACE '{print $2}'; cat "+path+" | awk 'BEGIN{find=0;}{for(i=1;i<=NF;i++){if($i==\"IFACE\"){j=i+1;if($j== \"rxpck/s\"){find=1;start_col=j;col=NF;for(k=1;k<=col;k++){res_arr[k]=0;}next};if($j==\"rxerr/s\"){find=0;for(k=start_col;k<=col;k++)printf res_arr[k]\"\"FS; print \"\";ne xt}}if($i==\"lo\")next;if(find){res_arr[i]+=$i}}}'") + result["nic"] = self.common.convert_table_to_2Dlist(stdout) + #4. tps + return result + + def process_iostat_data(self, node, path): + result = {} + output_list = [] + dict_diskformat = {} + if node in self.cluster["osds"]: + output_list = self.common.parse_disk_format( self.cluster['diskformat'] ) + for i in range(len(output_list)): + disk_list=[] + for osd_journal in self.common.get_list(self.all_conf_data.get_list(node)): + tmp_dev_name = osd_journal[i].split('/')[2] + if 'nvme' in tmp_dev_name: + tmp_dev_name = self.common.parse_nvme( tmp_dev_name ) + if tmp_dev_name not in disk_list: + disk_list.append( tmp_dev_name ) + dict_diskformat[output_list[i]]=disk_list + elif node in self.cluster["vclient"]: + vdisk_list = [] + for disk in self.cluster["vclient_disk"]: + vdisk_list.append( disk.split('/')[2] ) + output_list = ["vdisk"] + # get total second + runtime = self.common.bash("grep 'Device' "+path+" | wc -l ").strip() + for output in output_list: + if output != "vdisk": + disk_list = " ".join(dict_diskformat[output]) + disk_num = len(list(set(dict_diskformat[output]))) + else: + disk_list = " ".join(vdisk_list) + disk_num = len(vdisk_list) + stdout = self.common.bash( "grep 'Device' -m 1 "+path+" | awk -F\"Device:\" '{print $2}'; cat "+path+" | awk -v dev=\""+disk_list+"\" -v line="+runtime+" 'BEGIN{split(dev,dev_arr,\" \");dev_count=0;for(k in dev_arr){count[k]=0;dev_count+=1};for(i=1;i<=line;i++)for(j=1;j<=NF;j++){res_arr[i,j]=0}}{for(k in dev_arr)if(dev_arr[k]==$1){cur_line=count[k];for(j=2;j<=NF;j++){res_arr[cur_line,j]+=$j;}count[k]+=1;col=NF}}END{for(i=1;i<=line;i++){for(j=2;j<=col;j++)printf (res_arr[i,j]/dev_count)\"\"FS; print \"\"}}'") + result[output] = self.common.convert_table_to_2Dlist(stdout) + result[output]["disk_num"] = disk_num + return result + + def process_vdbench_data(self, path, dirname): + result = {} + vdbench_data = {} + runtime = int(self.common.bash("grep -o 'elapsed=[0-9]\+' "+path+" | cut -d = -f 2")) + stdout, stderr = self.common.bash("grep 'avg_2-' "+path, True) + vdbench_data = stdout.split() + output_vdbench_data = OrderedDict() + output_vdbench_data['read_lat'] = vdbench_data[8] + output_vdbench_data["read_iops"] = vdbench_data[7] + output_vdbench_data["read_bw"] = vdbench_data[11] + output_vdbench_data['read_runtime'] = runtime + output_vdbench_data['write_lat'] = vdbench_data[10] + output_vdbench_data["write_iops"] = vdbench_data[9] + output_vdbench_data["write_bw"] = vdbench_data[12] + output_vdbench_data['write_runtime'] = runtime + output_vdbench_data['lat_unit'] = 'msec' + output_vdbench_data['runtime_unit'] = 'sec' + output_vdbench_data['bw_unit'] = 'MB/s' + result[dirname] = {} + result[dirname]["vdbench"] = output_vdbench_data + return result + + def get_lat_persent_dict(self,fio_str): + lat_percent_dict = {} + tmp_list = fio_str.split(',') + for i in tmp_list: + li = i.split('=') + while '' in li:li.remove('') + if len(li) == 2 and li[1] != '': + key = re.findall('.*?th',li[0].strip('\n').strip('| ').strip(' ').replace(' ',''),re.S) + value = re.match(r'\[(.*?)\]',li[1].strip('\n').strip(' ').replace(' ','')).groups() + if len(key) != 0 and len(value) != 0: + lat_percent_dict[key[0]] = value[0] + return lat_percent_dict + + def process_fio_data(self, path, dirname): + result = {} + stdout = self.common.bash("grep \" *io=.*bw=.*iops=.*runt=.*\|^ *lat.*min=.*max=.*avg=.*stdev=.*\" "+path) + stdout1 = self.common.bash("grep \" *1.00th.*],\| *30.00th.*],\| *70.00th.*],\| *99.00th.*],\| *99.99th.*]\" "+path) + stdout2 = self.common.bash("grep \" *clat percentiles\" "+path) + + lat_per_dict = {} + if stdout1 != '': + lat_per_dict = self.get_lat_persent_dict(stdout1) + + fio_data_rw = {} + fio_data_rw["read"] = {} + fio_data_rw["write"] = {} + for data in re.split(',|\n|:',stdout): + try: + key, value = data.split("=") + if key.strip() not in fio_data: + fio_data[key.strip()] = [] + fio_data[key.strip()].append( value.strip() ) + except: + if 'lat' in data: + res = re.search('lat\s*\((\w+)\)',data) + if 'lat_unit' not in fio_data: + fio_data['lat_unit'] = [] + fio_data['lat_unit'].append( res.group(1) ) + if "read" in data: + fio_data = fio_data_rw["read"] + if "write" in data: + fio_data = fio_data_rw["write"] + + output_fio_data = OrderedDict() + output_fio_data['read_lat'] = 0 + output_fio_data['read_iops'] = 0 + output_fio_data['read_bw'] = 0 + output_fio_data['read_runtime'] = 0 + output_fio_data['write_lat'] = 0 + output_fio_data['write_iops'] = 0 + output_fio_data['write_bw'] = 0 + output_fio_data['write_runtime'] = 0 + if len(lat_per_dict) != 0: + if '99.99th' in lat_per_dict.keys(): + #output_fio_data['99.99%_lat'] = lat_per_dict['99.99th'] + lat_persent_unit = re.findall(r"(?<=[\(])[^\)]+(?=[\)])", stdout2.strip('\n').strip(' ').replace(' ','')) + if len(lat_persent_unit) != 0: + output_fio_data['99.99%_lat'] = float(self.common.time_to_sec("%s%s" % (lat_per_dict['99.99th'], lat_persent_unit[0]),'msec')) + else: + output_fio_data['99.99%_lat'] = 'null' + else: + output_fio_data['99.99%_lat'] = 'null' + output_fio_data['lat_unit'] = 'msec' + output_fio_data['runtime_unit'] = 'sec' + output_fio_data['bw_unit'] = 'MB/s' + for io_pattern in ['read', 'write']: + if fio_data_rw[io_pattern] != {}: + first_item = fio_data_rw[io_pattern].keys()[0] + else: + continue + list_len = len(fio_data_rw[io_pattern][first_item]) + for index in range(0, list_len): + fio_data = fio_data_rw[io_pattern] + output_fio_data['%s_lat' % io_pattern] += float(self.common.time_to_sec("%s%s" % (fio_data['avg'][index], fio_data['lat_unit'][index]),'msec')) + output_fio_data['%s_iops' % io_pattern] += int(fio_data['iops'][index]) + res = re.search('(\d+\.*\d*)\s*(\w+)/s',fio_data['bw'][index]) + if res: + output_fio_data['%s_bw' % io_pattern] += float( self.common.size_to_Kbytes("%s%s" % (res.group(1), res.group(2)),'MB') ) + output_fio_data['%s_runtime' % io_pattern] += float( self.common.time_to_sec(fio_data['runt'][index], 'sec') ) + output_fio_data['%s_lat' % io_pattern] /= list_len + output_fio_data['%s_runtime' % io_pattern] /= list_len + result[dirname] = {} + result[dirname]["fio"] = output_fio_data + return result + + def process_lttng_data(self, path): + pass + + def process_perf_data(self, path): + pass + + def process_blktrace_data(self, path): + pass + + def process_perfcounter_data(self, path): + precise_level = int(self.cluster["perfcounter_time_precision_level"]) +# precise_level = 6 + self.common.printout("LOG","loading %s" % path) + perfcounter = [] + with open(path,"r") as fd: + data = fd.readlines() + for tmp_data in data: + if ',' == tmp_data[-2]: + tmp_data = tmp_data[:-2] + try: + perfcounter.append(json.loads(tmp_data, object_pairs_hook=OrderedDict)) + except: + perfcounter.append({}) + if not len(perfcounter) > 0: + return False + result = self.common.MergableDict() + lastcounter = perfcounter[0] + for counter in perfcounter[1:]: + result.update(counter, dedup=False, diff=False) + result = result.get() + output = OrderedDict() +# for key in ["osd", "filestore", "objecter", "mutex-JOS::SubmitManager::lock"]: + for key in self.cluster["perfcounter_data_type"]: + result_key = key + find = True + if key != "librbd" and key not in result: + continue + if key == "librbd": + find = False + for result_key in result.keys(): + if key in result_key: + find = True + break + if not find: + continue + output["perfcounter_"+key] = {} + current = output["perfcounter_"+key] + for param, data in result[result_key].items(): + if isinstance(data, list): + if not param in current: + current[param] = [] + current[param].extend( data ) + if isinstance(data, dict) and 'avgcount' in data and 'sum' in data: + if not isinstance(data['sum'], list): + continue + if not param in current: + current[param] = [] + last_sum = data['sum'][0] + last_avgcount = data['avgcount'][0] + for i in range(1, len(data['sum'])): + try: + current[param].append( round((data['sum'][i]-last_sum)/(data['avgcount'][i]-last_avgcount),precise_level) ) + except: + current[param].append(0) + last_sum = data['sum'][i] + last_avgcount = data['avgcount'][i] + return output + +def main(args): + parser = argparse.ArgumentParser(description='Analyzer tool') + parser.add_argument( + 'operation', + ) + parser.add_argument( + '--path', + ) + parser.add_argument( + '--path_detail', + ) + parser.add_argument( + '--node', + ) + parser.add_argument( + '--name', + ) + args = parser.parse_args(args) + process = Analyzer(args.path,args.name) + if args.operation == "process_data": + process.process_data() + else: + func = getattr(process, args.operation) + if func: + func(args.path_detail) + +if __name__ == '__main__': + import sys + main(sys.argv[1:]) diff --git a/benchmarking/.run_number b/benchmarking/.run_number index 0947c33..00750ed 100644 --- a/benchmarking/.run_number +++ b/benchmarking/.run_number @@ -1 +1 @@ -188 \ No newline at end of file +3 diff --git a/benchmarking/mod/bblock/__init__.py b/benchmarking/mod/bblock/__init__.py index affda02..07dd2f1 100644 --- a/benchmarking/mod/bblock/__init__.py +++ b/benchmarking/mod/bblock/__init__.py @@ -1,3 +1,3 @@ -__all__ = ['qemurbd', 'fiorbd'] +__all__ = ['qemurbd', 'fiorbd', 'vdbench'] diff --git a/benchmarking/mod/bblock/fiorbd.py b/benchmarking/mod/bblock/fiorbd.py index e94ffda..f8299a8 100644 --- a/benchmarking/mod/bblock/fiorbd.py +++ b/benchmarking/mod/bblock/fiorbd.py @@ -5,7 +5,7 @@ class FioRbd(Benchmark): def load_parameter(self): super(self.__class__, self).load_parameter() - self.cluster["rbdlist"] = self.get_rbd_list() + self.cluster["rbdlist"] = self.get_rbd_list(self.benchmark["poolname"]) if len(self.cluster["rbdlist"]) < int(self.all_conf_data.get("rbd_volume_count")): self.prepare_images() @@ -21,13 +21,13 @@ def prepare_images(self): rbd_size = self.all_conf_data.get("volume_size") common.printout("LOG","Creating rbd volume") if rbd_count and rbd_size: - super(self.__class__, self).create_image(rbd_count, rbd_size, 'rbd') + super(self.__class__, self).create_image(rbd_count, rbd_size, self.benchmark["poolname"]) else: common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf") - #start to init + #start to init dest_dir = self.cluster["tmp_dir"] disk_num_per_client = self.cluster["disk_num_per_client"] - self.cluster["rbdlist"] = self.get_rbd_list() + self.cluster["rbdlist"] = self.get_rbd_list(self.benchmark["poolname"]) instance_list = self.cluster["rbdlist"] self.testjob_distribution(disk_num_per_client, instance_list) fio_job_num_total = 0 @@ -35,8 +35,9 @@ def prepare_images(self): for client in self.cluster["testjob_distribution"]: common.scp(user, client, "../conf/fio_init.conf", dest_dir) rbdlist = ' '.join(self.cluster["testjob_distribution"][client]) - res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --section init-write %s/fio_init.conf & done" % (rbdlist, 'rbd', dest_dir), option = "force") + res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --section init-write %s/fio_init.conf & done" % (rbdlist, self.benchmark["poolname"], dest_dir), option = "force") fio_job_num_total += len(self.cluster["testjob_distribution"][client]) + common.printout("LOG","%d FIO Jobs starts on %s" % (len(self.cluster["testjob_distribution"][client]), client)) time.sleep(1) if not self.check_fio_pgrep(clients, fio_job_num_total): common.printout("ERROR","Failed to start FIO process") @@ -45,7 +46,6 @@ def prepare_images(self): if not fio_job_num_total: common.printout("ERROR","Planed to run 0 Fio Job, please check all.conf") raise KeyboardInterrupt - common.printout("LOG","%d FIO Jobs starts on %s" % (len(self.cluster["testjob_distribution"][client]), client)) common.printout("LOG","Wait rbd initialization stop") #wait fio finish @@ -60,7 +60,7 @@ def prepare_images(self): def prepare_result_dir(self): #1. prepare result dir self.get_runid() - self.benchmark["section_name"] = "fiorbd-%s-%s-qd%s-%s-%s-%s-fiorbd" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"]) + self.benchmark["section_name"] = "fiorbd-%s-%s-qd%s-%s-%s-%s-%s" % (self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"], self.benchmark["poolname"]) self.benchmark["dirname"] = "%s-%s-%s" % (str(self.runid), str(self.benchmark["instance_number"]), self.benchmark["section_name"]) self.cluster["dest_dir"] = "/%s/%s" % (self.cluster["dest_dir"], self.benchmark["dirname"]) super(self.__class__, self).prepare_result_dir() @@ -90,9 +90,10 @@ def run(self): nodes = self.benchmark["distribution"].keys() fio_job_num_total = 0 + poolname = self.benchmark["poolname"] for client in self.benchmark["distribution"]: rbdlist = ' '.join(self.benchmark["distribution"][client]) - res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --output %s/`hostname`_${rbdname}_fio.txt --write_bw_log=%s/`hostname`_${rbdname}_fio --write_lat_log=%s/`hostname`_${rbdname}_fio --write_iops_log=%s/`hostname`_${rbdname}_fio --section %s %s/fio.conf 2>%s/`hostname`_${rbdname}_fio_errorlog.txt & done" % (rbdlist, 'rbd', dest_dir, dest_dir, dest_dir, dest_dir, self.benchmark["section_name"], dest_dir, dest_dir), option = "force") + res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --output %s/`hostname`_${rbdname}_fio.txt --write_bw_log=%s/`hostname`_${rbdname}_fio --write_lat_log=%s/`hostname`_${rbdname}_fio --write_iops_log=%s/`hostname`_${rbdname}_fio --section %s %s/fio.conf 2>%s/`hostname`_${rbdname}_fio_errorlog.txt & done" % (rbdlist, poolname, dest_dir, dest_dir, dest_dir, dest_dir, self.benchmark["section_name"], dest_dir, dest_dir), option = "force") fio_job_num_total += len(self.benchmark["distribution"][client]) self.chkpoint_to_log("fio start") time.sleep(1) @@ -152,8 +153,9 @@ def generate_benchmark_cases(self, testcase): runtime = testcase["runtime"] disk = testcase["vdisk"] description = testcase["description"] + poolname = testcase["poolname"] - fio_list = [] + fio_list = [] fio_list.append("[global]") fio_list.append(" direct=1") fio_list.append(" time_based") @@ -164,7 +166,7 @@ def generate_benchmark_cases(self, testcase): io_pattern_fio = "write" fio_template = [] - fio_template.append("[fiorbd-%s-%s-qd%s-%s-%s-%s-fiorbd]" % (io_pattern, record_size, queue_depth, rbd_volume_size, warmup_time, runtime )) + fio_template.append("[fiorbd-%s-%s-qd%s-%s-%s-%s-%s]" % (io_pattern, record_size, queue_depth, rbd_volume_size, warmup_time, runtime, poolname )) fio_template.append(" rw=%s" % io_pattern_fio) fio_template.append(" bs=%s" % record_size) fio_template.append(" iodepth=%s" % queue_depth) @@ -212,10 +214,10 @@ def parse_benchmark_cases(self, testcase): testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], "block_size":p[3], "qd":p[4], "rampup":p[5], - "runtime":p[6], "vdisk":p[7] + "runtime":p[6], "vdisk":p[7], "poolname":p[8] } - if len(p) == 9: - testcase_dict["description"] = p[8] + if len(p) >= 10: + testcase_dict["description"] = p[9] else: testcase_dict["description"] = "" return testcase_dict diff --git a/benchmarking/mod/bblock/qemurbd.py b/benchmarking/mod/bblock/qemurbd.py index d296a7f..76f2f1f 100644 --- a/benchmarking/mod/bblock/qemurbd.py +++ b/benchmarking/mod/bblock/qemurbd.py @@ -327,10 +327,10 @@ def parse_benchmark_cases(self, testcase): testcase_dict = { "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], "block_size":p[3], "qd":p[4], "rampup":p[5], - "runtime":p[6], "vdisk":p[7] + "runtime":p[6], "vdisk":p[7], "poolname":p[8] } - if len(p) == 9: - testcase_dict["description"] = p[8] + if len(p) >= 10: + testcase_dict["description"] = p[9] else: testcase_dict["description"] = "" diff --git a/benchmarking/mod/bblock/vdbench.py b/benchmarking/mod/bblock/vdbench.py new file mode 100644 index 0000000..a6a95a5 --- /dev/null +++ b/benchmarking/mod/bblock/vdbench.py @@ -0,0 +1,307 @@ +from ..benchmark import * +from collections import OrderedDict +import itertools + +class VdBench(Benchmark): + def __init__(self): + self.bench_type = "vdbench" + super(self.__class__, self).__init__() + self.cluster["bench_dir"] = "%s/%s/" % (self.all_conf_data.get("tmp_dir"), self.bench_type) + # Format default output dir: vdbench/output/ + self.cluster["format_output_dir"] = "%s/output/" % (self.cluster["bench_dir"]) + # Run results dir: vdbench/results/ + self.cluster["result_dir"] = "%s/results/" % (self.cluster["bench_dir"]) + common.printout("LOG","bench dir: %s, format output dir: %s, result dir: %s" % (self.cluster["bench_dir"], self.cluster["format_output_dir"], self.cluster["result_dir"])) + + def load_parameter(self): + super(self.__class__, self).load_parameter() + self.custom_script = self.all_conf_data.get("custom_script", True ) + + self.cluster["vclient"] = self.all_conf_data.get_list("list_vclient") + disk_num_per_client = self.cluster["disk_num_per_client"] + self.disk_num_per_client = disk_num_per_client + self.volume_size = self.all_conf_data.get("volume_size") + self.instance_list = self.cluster["vclient"] + self.testjob_distribution(disk_num_per_client, self.instance_list) + + def cal_run_job_distribution(self): + number = int(self.benchmark["instance_number"]) + client_total = len(self.cluster["client"]) + # Assume number is always 50 here + self.benchmark["distribution"] = {} + client_num = 0 + for client in self.cluster["testjob_distribution"]: + vclient_total = int(self.disk_num_per_client[client_num]) + self.benchmark["distribution"][client] = copy.deepcopy(self.cluster["testjob_distribution"][client][:vclient_total]) + client_num += 1 + nodes = [] + for client in self.benchmark["distribution"]: + nodes.extend(self.benchmark["distribution"][client]) + self.cluster["nodes_distribution"] = nodes + + def prepare_result_dir(self): + #1. prepare result dir + self.get_runid() + vdisk = self.benchmark["vdisk"].split('/')[-1] + self.benchmark["section_name"] = "%s-%s-%s-qd%s-%s-%s-%s-%s" % (self.bench_type, self.benchmark["iopattern"], self.benchmark["block_size"], self.benchmark["qd"], self.benchmark["volume_size"],self.benchmark["rampup"], self.benchmark["runtime"], vdisk) + self.benchmark["dirname"] = "%s-%s-%s" % (str(self.runid), str(self.benchmark["instance_number"]), self.benchmark["section_name"]) + self.cluster["dest_dir"] = "/%s/%s" % (self.cluster["dest_dir"], self.benchmark["dirname"]) + + res = common.pdsh(self.cluster["user"],["%s"%(self.cluster["head"])],"test -d %s" % (self.cluster["dest_dir"]), option = "check_return") + if not res[1]: + common.printout("ERROR","Output DIR %s exists" % (self.cluster["dest_dir"])) + sys.exit() + + common.pdsh(self.cluster["user"] ,["%s" % (self.cluster["head"])], "mkdir -p %s" % (self.cluster["dest_dir"])) + + def cleanup(self): + super(self.__class__, self).cleanup() + #1. clean the tmp res dir + user = self.cluster["user"] + nodes = self.cluster["nodes_distribution"] + common.pdsh(user, nodes, "rm -rf %s/*" % self.cluster["format_output_dir"]) + common.pdsh(user, nodes, "rm -rf %s/*" % self.cluster["result_dir"]) + + def check_run_success(self, check_file, max_time): + user = self.cluster["user"] + nodes = self.cluster["nodes_distribution"] + cur_check = 0 + sleep_sec = 2 + max_check = max_time / sleep_sec + while cur_check < max_check: + common.printout("LOG", "checking... %s" % cur_check) + stdout, stderr = common.pdsh(user, nodes, "grep completed %s" % check_file, option="check_return") + res = common.format_pdsh_return(stdout) + if len(nodes) != len(res.keys()): + time.sleep(sleep_sec) + else: + common.printout("LOG", "checking done") + return + cur_check += 1 + common.printout("ERROR","Checking run in %s failed" % check_file) + stdout, stderr = common.pdsh(user, nodes, "grep -q completed %s; if [ $? -ne 0 ]; then echo Run is not completed successfully; fi" % check_file, option="check_return") + sys.exit() + + def format_run(self): + common.printout("LOG", "Start Formatting!") + user = self.cluster["user"] + nodes = self.cluster["nodes_distribution"] + common.pdsh(user, nodes, "cd %s; ./vdbench -f format.cfg -o %s" % (self.cluster["bench_dir"], self.cluster["format_output_dir"])) + check_file = "%s/summary.html" % self.cluster["format_output_dir"] + self.check_run_success(check_file, 100) + + def prepare_run(self): + super(self.__class__, self).prepare_run() + user = self.cluster["user"] + dest_dir = self.cluster["tmp_dir"] + self.cleanup() + # format + self.format_run() + + def wait_workload_to_stop(self): + pass + + def stop_workload(self): + pass + + def generate_benchmark_cases(self, testcase): + io_pattern = testcase["iopattern"] + block_size = testcase["block_size"] + queue_depth = testcase["qd"] + rbd_volume_size = testcase["volume_size"] + warmup_time = testcase["rampup"] + runtime = int(testcase["runtime"]) + disk = testcase["vdisk"] + + custom_params = testcase["custom_parameters"] + for str in custom_params.split(','): + str2 = str.split('=') + if len(str2) != 2: + continue + if str2[0] == "width": + width = int(str2[1]); + elif str2[0] == "depth": + depth = int(str2[1]); + elif str2[0] == "files": + files_num = int(str2[1]); + elif str2[0] == "threads": + threads_num = int(str2[1]); + elif str2[0] == "rdpct": + read_percentage = int(str2[1]); + + if int(re.findall(r"\d", block_size)[0]) * depth * width > int(re.findall(r"\d", rbd_volume_size)[0]) * 1024 * 1024: + common.printout("ERROR","Files total size is too big, bigger than volume size!") + raise KeyboardInterrupt + + if io_pattern in ["randread", "randwrite", "randrw"]: + fileio = "random" + if io_pattern in ["seqread", "seqwrite", "readwrite", "rw"]: + fileio = "sequential" + if io_pattern == "randread": + read_percentage = 100 + if io_pattern == "randwrite": + read_percentage = 0 + + format_cfg = [] + format_cfg.append("fsd=fsd1,anchor=/mnt/,depth=%d,width=%d,files=%d,size=%s" % (depth, width, files_num, block_size)) + format_cfg.append("fwd=default") + format_cfg.append("fwd=fwd1,fsd=fsd1") + format_cfg.append("rd=rd0,fwd=fwd1,fwdrate=max,format=only") + with open("../conf/format.cfg", "w+") as f: + f.write("\n".join(format_cfg)+"\n") + + case_cfg = [] + case_cfg.append("fsd=fsd1,anchor=/mnt/,depth=%d,width=%d,files=%d,size=%s" % (depth, width, files_num, block_size)) + case_cfg.append("fwd=default,xfersize=4k,fileio=%s,fileselect=random,threads=%d" % (fileio, threads_num)) + case_cfg.append("fwd=fwd1,fsd=fsd1,rdpct=%d" % read_percentage) + case_cfg.append("rd=rd1,fwd=fwd1,fwdrate=max,format=no,elapsed=%d,interval=1" % runtime) + with open("../conf/vdbench_test.cfg", "w+") as f: + f.write("\n".join(case_cfg)+"\n") + + params_list = [] + params_list.append("depth=%d,width=%d,files=%d,threads=%d,rdpct=%d" % (depth, width, files_num, threads_num, read_percentage)) + with open("../conf/vdbench_params.txt", "w+") as f: + f.write("\n".join(params_list)+"\n") + return True + + def parse_benchmark_cases(self, testcase): + p = testcase + testcase_dict = { + "instance_number":p[0], "volume_size":p[1], "iopattern":p[2], + "block_size":p[3], "qd":p[4], "rampup":p[5], + "runtime":p[6], "vdisk":p[7], "custom_parameters":p[8] + } + if len(p) == 10: + testcase_dict["description"] = p[9] + elif len(p) > 10: + common.printout("ERROR","Too much columns found for test case ") + sys.exit() + else: + testcase_dict["description"] = "" + + return testcase_dict + + def run(self): + super(self.__class__, self).run() + user = self.cluster["user"] + nodes = self.cluster["nodes_distribution"] + waittime = 15 + common.printout("LOG", "Start Running VdBench!") + common.pdsh(user, nodes, "cd %s; ./vdbench -f vdbench_test.cfg -o %s" % (self.cluster["bench_dir"], self.cluster["result_dir"])) + check_file = "%s/summary.html" % self.cluster["result_dir"] + self.check_run_success(check_file, 100) + for wait in range(1, waittime): + time.sleep(1) + + def archive(self): + super(self.__class__, self).archive() + user = self.cluster["user"] + nodes = self.cluster["nodes_distribution"] + head = self.cluster["head"] + dest_dir = self.cluster["dest_dir"] + common.cp("%s/conf/vdbench_params.txt" % self.pwd, "%s/conf/" % dest_dir) + #collect client data + for node in nodes: + common.bash("mkdir -p %s/raw/%s" % (dest_dir, node)) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*" % self.cluster["result_dir"]) + + + def prepare_images(self): + user = self.cluster["user"] + dest_dir = self.cluster["tmp_dir"] + controller = self.cluster["head"] + rbd_count = len(self.instance_list) + rbd_size = self.all_conf_data.get("volume_size") + if rbd_count and rbd_size: + super(self.__class__, self).create_image(rbd_count, rbd_size, 'rbd') + else: + common.printout("ERROR","need to set rbd_volume_count and volune_size in all.conf") + + #create image xml + common.printout("LOG","create rbd volume vm attach xml") + common.scp(user, controller, "%s/vm-scripts" % (self.pwd), "/opt/"); + common.scp(user, controller, "%s/conf" % (self.pwd), "/opt/"); + common.pdsh(user, [controller], "cd /opt/vm-scripts; echo 3 | bash create-volume.sh create_disk_xml", "check_return") + common.rscp(user, controller, "%s/vm-scripts/" % (self.pwd), "/opt/vm-scripts/vdbs/"); + common.printout("LOG","Distribute vdbs xml") + for client in self.cluster["testjob_distribution"]: + common.scp(user, client, "../vm-scripts/vdbs", dest_dir) + + #attach to vm + self.attach_images(self.cluster["testjob_distribution"]) + + #start to init + common.printout("LOG","rbd initialization finished") + + def prepare_case(self, user, nodes): + stdout, stderr = common.pdsh(user, nodes, "test -d %s" % self.cluster["bench_dir"], option="check_return") + if stderr: + common.printout("LOG","Distribute vdbench benchmark execution file") + for node in nodes: + common.scp(user, node, '../conf/%s.tar.gz' % self.bench_type, '%s' % self.cluster["tmp_dir"]) + common.pdsh(user, nodes, 'cd %s; tar xzf %s.tar.gz' % (self.cluster["tmp_dir"], self.bench_type)) + + common.pdsh(user, nodes, "mkdir -p %s" % self.cluster["result_dir"]) + for node in nodes: + common.scp(user, node, "../conf/format.cfg", "%s" % self.cluster["bench_dir"]) + common.scp(user, node, "../conf/vdbench_test.cfg", "%s" % self.cluster["bench_dir"]) + + def prerun_check(self): + super(self.__class__, self).prerun_check() + #1. check is vclient alive + user = self.cluster["user"] + vdisk = self.benchmark["vdisk"] + nodes = self.cluster["nodes_distribution"] + planed_space = str(len(self.instance_list) * int(self.volume_size)) + "MB" + common.printout("LOG","Prerun_check: check if rbd volume be intialized") + if not self.check_rbd_init_completed(planed_space): + common.printout("WARNING","rbd volume initialization has not be done") + self.prepare_images() + + common.printout("LOG","Distribution nodes: %s" % nodes) + self.prepare_case(user, nodes) + + common.printout("LOG","Prerun_check: check if rbd volume attached") + need_to_attach = False + stdout, stderr = common.pdsh(user, nodes, "fdisk -l %s" % vdisk, option="check_return") + res = common.format_pdsh_return(stdout) + if len(nodes) != len(res.keys()): + need_to_attach = True + if need_to_attach: + common.printout("WARNING","vclients are not attached with rbd volume") + self.attach_images() + common.printout("WARNING","vclients attached rbd volume now") + common.printout("LOG","Prerun_check: check if sysstat installed on %s" % nodes) + common.pdsh(user, nodes, "mpstat") + common.pdsh(user, nodes, "killall -9 java", option = "check_return") + + def attach_images(self, to_attach_dict = None): + user = self.cluster["user"] + vdisk = self.benchmark["vdisk"] + dest_dir = self.cluster["tmp_dir"] + if not to_attach_dict: + to_attach_dict = self.benchmark["distribution"] + for client in to_attach_dict: + nodes = to_attach_dict[client] + for node in nodes: + common.printout("LOG","Attach rbd image to %s" % node) + stdout, stderr = common.pdsh(user, [node], "fdisk -l %s" % vdisk, option="check_return") + res = common.format_pdsh_return(stdout) + if node not in res: + common.pdsh(user, [client], "cd %s/vdbs; virsh attach-device %s %s.xml" % (dest_dir, node, node), except_returncode=1) + common.pdsh(user, [node], "mount | grep /dev/vdb1; if [ $? ne 0]; then parted -s -a optimal /dev/vdb mklabel gpt -- mkpart primary ext4 1 100%; mkfs -t ext4 /dev/vdb1; mount /dev/vdb1 /mnt; fi") + + def detach_images(self): + user = self.cluster["user"] + vdisk = self.benchmark["vdisk"] + tmp_vdisk = re.search('/dev/(\w+)',vdisk) + vdisk_suffix = tmp_vdisk.group(1) + #for client in self.cluster["testjob_distribution"]: + for client in self.benchmark["distribution"]: + nodes = self.benchmark["distribution"][client] + for node in nodes: + common.printout("LOG","Detach rbd image from %s" % node) + stdout, stderr = common.pdsh(user, [node], "df %s" % vdisk, option="check_return") + if not stderr: + common.pdsh(user, [client], "virsh detach-disk %s %s" % (node, vdisk_suffix), except_returncode=1) + diff --git a/benchmarking/mod/benchmark.py b/benchmarking/mod/benchmark.py index e66bbfe..5241629 100644 --- a/benchmarking/mod/benchmark.py +++ b/benchmarking/mod/benchmark.py @@ -19,11 +19,14 @@ def __init__(self): def go(self, testcase, tuning): common.bash("rm -f %s/conf/%s" % (self.pwd, common.cetune_log_file)) common.bash("rm -f %s/conf/%s" % (self.pwd, common.cetune_error_file)) + user = self.all_conf_data.get("user") + controller = self.all_conf_data.get("head") + common.wait_ceph_to_health( user, controller ) + self.benchmark = self.parse_benchmark_cases(testcase) self.load_parameter() self.get_runid() self.set_runid() - self.benchmark = self.parse_benchmark_cases(testcase) if not self.generate_benchmark_cases(self.benchmark): common.printout("ERROR", "Failed to generate Fio/cosbench configuration file.") sys.exit() @@ -64,7 +67,7 @@ def go(self, testcase, tuning): def create_image(self, volume_count, volume_size, poolname): user = self.cluster["user"] controller = self.cluster["head"] - rbd_list = self.get_rbd_list() + rbd_list = self.get_rbd_list(poolname) need_to_create = 0 if not len(rbd_list) >= int(volume_count): need_to_create = int(volume_count) - len(rbd_list) @@ -74,10 +77,9 @@ def create_image(self, volume_count, volume_size, poolname): common.pdsh(user, [controller], "rbd create -p %s --size %s %s --image-format 2" % (poolname, str(volume_size), volume)) common.printout("LOG","%d RBD Image Created" % need_to_create) - def get_rbd_list(self): + def get_rbd_list(self, poolname): user = self.cluster["user"] controller = self.cluster["head"] - poolname = "rbd" stdout, stderr = common.pdsh(user, [controller], "rbd ls -p %s" % poolname, option="check_return") if stderr: common.printout("ERROR","unable get rbd list, return msg: %s" % stderr) @@ -163,6 +165,7 @@ def run(self): common.pdsh(user, nodes, "mpstat -P ALL %s > %s/`hostname`_mpstat.txt & echo `date +%s`' mpstat start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) common.pdsh(user, nodes, "iostat -p ALL -dxm %s > %s/`hostname`_iostat.txt & echo `date +%s`' iostat start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) common.pdsh(user, nodes, "sar -A %s > %s/`hostname`_sar.txt & echo `date +%s`' sar start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) + common.pdsh(user, nodes, "ceph -v >> %s/`hostname`_ceph_version.txt" % (dest_dir)) if "perfcounter" in self.cluster["collector"]: common.printout("LOG","Start perfcounter data collector under %s " % nodes) common.pdsh(user, nodes, "echo `date +%s`' perfcounter start' >> %s/`hostname`_process_log.txt; for i in `seq 1 %d`; do find /var/run/ceph -name '*osd*asok' | while read path; do filename=`echo $path | awk -F/ '{print $NF}'`;res_file=%s/`hostname`_${filename}.txt; echo `ceph --admin-daemon $path perf dump`, >> ${res_file} & done; sleep %s; done; echo `date +%s`' perfcounter stop' >> %s/`hostname`_process_log.txt;" % ('%s', dest_dir, time_tmp, dest_dir, monitor_interval, '%s', dest_dir), option="force") @@ -217,6 +220,7 @@ def run(self): common.pdsh(user, nodes, "mpstat -P ALL %s > %s/`hostname`_mpstat.txt & echo `date +%s`' mpstat start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) common.pdsh(user, nodes, "iostat -p -dxm %s > %s/`hostname`_iostat.txt & echo `date +%s`' iostat start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) common.pdsh(user, nodes, "sar -A %s > %s/`hostname`_sar.txt & echo `date +%s`' sar start' >> %s/`hostname`_process_log.txt" % (monitor_interval, dest_dir, '%s', dest_dir)) + common.pdsh(user, nodes, "ceph -v >> %s/`hostname`_ceph_version.txt" % (dest_dir)) if "perfcounter" in self.cluster["collector"]: common.printout("LOG","Start perfcounter data collector under %s " % nodes) common.pdsh(user, nodes, "echo `date +%s`' perfcounter start' >> %s/`hostname`_process_log.txt; for i in `seq 1 %d`; do find /var/run/ceph -name '*client*asok' | while read path; do filename=`echo $path | awk -F/ '{print $NF}'`;res_file=%s/`hostname`_${filename}.txt; echo `ceph --admin-daemon $path perf dump`, >> ${res_file} & done; sleep %s; done; echo `date +%s`' perfcounter stop' >> %s/`hostname`_process_log.txt;" % ('%s', dest_dir, time_tmp, dest_dir, monitor_interval, '%s', dest_dir), option="force") @@ -248,6 +252,8 @@ def archive(self): for node in self.cluster["osd"]: common.bash("mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) if "blktrace" in self.cluster["collector"]: common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*blktrace*" % self.cluster["tmp_dir"]) if "lttng" in self.cluster["collector"]: @@ -257,6 +263,8 @@ def archive(self): for node in self.benchmark["distribution"].keys(): common.bash( "mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) #save real runtime if self.real_runtime: diff --git a/benchmarking/mod/generic/generic.py b/benchmarking/mod/generic/generic.py index 82bcc02..92b49ba 100644 --- a/benchmarking/mod/generic/generic.py +++ b/benchmarking/mod/generic/generic.py @@ -205,8 +205,8 @@ def parse_benchmark_cases(self, testcase): "block_size":p[3], "qd":p[4], "rampup":p[5], "runtime":p[6], "vdisk":p[7] } - if len(p) == 9: - testcase_dict["description"] = p[8] + if len(p) >= 10: + testcase_dict["description"] = p[9] else: testcase_dict["description"] = "" return testcase_dict diff --git a/benchmarking/run_cases.py b/benchmarking/run_cases.py index 9d809f7..e8a78c6 100644 --- a/benchmarking/run_cases.py +++ b/benchmarking/run_cases.py @@ -2,6 +2,7 @@ import os, sys lib_path = ( os.path.dirname(os.path.dirname(os.path.abspath(__file__)) )) +this_file_path = os.path.dirname(os.path.abspath(__file__)) sys.path.append(lib_path) from conf import * @@ -10,6 +11,8 @@ from mod.bobject import * from mod.bcephfs import * from mod.generic import * +from deploy import * +from tuner import * def main(args): parser = argparse.ArgumentParser(description='Cephperf Benchmark Script.') @@ -59,9 +62,12 @@ def main(args): else: with open("../conf/cases.conf", "r") as f: - for line in f.readlines(): + case_lines = f.readlines() + for line in case_lines: p = line.split() - testcase_list.append({"engine":p[0],"parameter":p[1:]}) + if len(p) > 0 and p!="\n": + if not p[0].startswith('#'): + testcase_list.append({"engine":p[0],"parameter":p[1:]}) for testcase in testcase_list: if testcase["engine"] == "qemurbd": benchmark = qemurbd.QemuRbd() @@ -75,9 +81,24 @@ def main(args): benchmark = generic.Generic() if testcase["engine"] == "hook": benchmark = hook.Hook() + if testcase["engine"] == "vdbench": + benchmark = vdbench.VdBench() if not benchmark: common.printout("ERROR","Unknown benchmark engine") try: + additional_option = '' + for i in testcase["parameter"]: + if i in ["restart","redeploy","resetPerf"]: + additional_option = i + if additional_option != '': + if additional_option == "restart": + run_deploy.main(['restart']) + if additional_option == "redeploy": + run_deploy.main(['redeploy']) + tuner.main(['--section', tuning_section,'apply_tuning']) + if additional_option == "resetPerf": + run_deploy.main(['osd_perf_reset']) + benchmark.go(testcase["parameter"], tuning_section) except KeyboardInterrupt: common.printout("WARNING","Caught KeyboardInterrupt Interruption") diff --git a/conf/all.conf.sample b/conf/all.conf.sample index 71ec2a2..034d8ab 100644 --- a/conf/all.conf.sample +++ b/conf/all.conf.sample @@ -10,6 +10,8 @@ journal_partition_size=30G rgw_server=rgw rgw_start_index=1 rgw_num_per_server=5 +distributed=1 +#0:local,1:remote # #=====================VCLIENT======================== list_vclient=vclient01,vclient02,vclient03,vclient04,vclient05,vclient06,vclient07,vclient08,vclient09,vclient10,vclient11,vclient12,vclient13,vclient14,vclient15,vclient16,vclient17,vclient18,vclient19,vclient20,vclient21,vclient22,vclient23,vclient24,vclient25,vclient26,vclient27,vclient28,vclient29,vclient30,vclient31,vclient32,vclient33,vclient34,vclient35,vclient36,vclient37,vclient38,vclient39,vclient40,vclient41,vclient42,vclient43,vclient44,vclient45,vclient46,vclient47,vclient48,vclient49,vclient50,vclient51,vclient52,vclient53,vclient54,vclient55,vclient56,vclient57,vclient58,vclient59,vclient60,vclient61,vclient62,vclient63,vclient64,vclient65,vclient66,vclient67,vclient68,vclient69,vclient70,vclient71,vclient72,vclient73,vclient74,vclient75,vclient76,vclient77,vclient78,vclient79,vclient80 diff --git a/conf/common.py b/conf/common.py index 0db6217..781d2b3 100644 --- a/conf/common.py +++ b/conf/common.py @@ -57,7 +57,19 @@ def addressToNetwork(self,ip,net): def getIpByHostInSubnet(self, hostname, subnet ): "Get IP by hostname and filter with subnet" - (hostname, aliaslist, ipaddrlist) = socket.gethostbyname_ex(hostname) + stdout, stderr = pdsh('root', [hostname] ,"ifconfig", option = "check_return") + if len(stderr): + printout("ERROR", 'Error to get ips: %s' % stderr) + sys.exit() + ipaddrlist = [] + res = re.findall("inet addr:\d+\.\d+\.\d+\.\d+",stdout) + for item in res: + b = item.split(':') + if b[1] != "127.0.0.1": + ipaddrlist.append(b[1]) + if len(ipaddrlist) == 0: + printout("ERROR", "No IP found") + sys.exit() try: network, netmask = self.networkMask(subnet) except: @@ -145,7 +157,7 @@ def pdsh(user, nodes, command, option="error_check", except_returncode=0, nodie= for node in nodes: _nodes.append("%s@%s" % (user, node)) _nodes = ",".join(_nodes) - args = ['pdsh', '-R', 'exec', '-w', _nodes, 'ssh', '%h', '-oConnectTimeout=15', command] + args = ['pdsh', '-R', 'exec', '-w', _nodes, '-f', str(len(nodes)), 'ssh', '%h', '-oConnectTimeout=15', command] # args = ['pdsh', '-w', _nodes, command] printout("CONSOLE", args, screen=False) @@ -412,9 +424,11 @@ def get(self): def size_to_Kbytes(size, dest_unit='KB'): if not str(size).isdigit(): - res = re.search('(\d+\.*\d*)\s*(\w+)',size) + res = re.search('(\d+\.*\d*)\s*(\D*)',size) space_num = float(res.group(1)) space_unit = res.group(2) + if space_unit == "": + space_unit = 'B' else: space_num = float(size) space_unit = 'B' @@ -506,6 +520,31 @@ def eval_args( obj, function_name, args ): res = func( **argv ) return res +def wait_ceph_to_health( user, controller ): + #wait ceph health to be OK + waitcount = 0 + try: + while not check_health( user, controller ) and waitcount < 300: + printout("WARNING","Applied tuning, waiting ceph to be healthy") + time.sleep(3) + waitcount += 3 + except: + printout("WARNING","Caught KeyboardInterrupt, exit") + sys.exit() + if waitcount < 300: + printout("LOG","Tuning has applied to ceph cluster, ceph is Healthy now") + else: + printout("ERROR","ceph is unHealthy after 300sec waiting, please fix the issue manually") + sys.exit() + +def check_health( user, controller ): + check_count = 0 + stdout, stderr = pdsh(user, [controller], 'ceph health', option="check_return") + if "HEALTH_OK" in stdout: + return True + else: + return False + def get_ceph_health(user, node): check_count = 0 output = {} diff --git a/conf/config.py b/conf/config.py index fd6490c..8fe7335 100644 --- a/conf/config.py +++ b/conf/config.py @@ -5,8 +5,12 @@ lib_path = os.path.abspath(os.path.join('..')) sys.path.append(lib_path) from collections import OrderedDict -from conf import common -from conf import description +try: + from conf import common + from conf import description +except: + import common + import description import re import argparse @@ -212,6 +216,7 @@ def check_config(self, key, value): required["monitoring_interval"] = {"type":"int"} required["disk_format"] = {"type":"diskformat"} required["disable_tuning_check"] = {"type":"bool"} + required["distributed_data_process"] = {"type":"bool"} helper = ConfigHelper() if key in required: @@ -261,20 +266,23 @@ def __init__(self): self.default_conf_path = "../conf/cases.default.conf" def set_config(self, case_json_list): + print "=======================set_config==========================" testcase_keys = [ "benchmark_driver","worker", "container_size", "iopattern", - "op_size", "object_size/QD", "rampup", "runtime", "device", "parameter", "desc" + "op_size", "object_size/QD", "rampup", "runtime", "device", "parameter", "desc","additional_option" ] case_list = [] for tmp_dict in json.loads(case_json_list): tmp = [] for key in testcase_keys: + if tmp_dict[key] == "": + tmp_dict[key] = "NULL" tmp.append(tmp_dict[key]) if tmp not in case_list: case_list.append(tmp) output = "" for case_items in case_list: - output += '%8s\t%4s\t%16s\t%8s\t%8s\t%16s\t%8s\t%8s\t%8s\t%8s\t%s\n' % ( case_items[0],case_items[1], case_items[2], case_items[3], case_items[4], case_items[5], case_items[6], case_items[7], case_items[8], case_items[9] ,case_items[10]) + output += '%8s\t%4s\t%16s\t%8s\t%8s\t%16s\t%8s\t%8s\t%8s\t%8s\t%s\t%6s\n' % ( case_items[0],case_items[1], case_items[2], case_items[3], case_items[4], case_items[5], case_items[6], case_items[7], case_items[8], case_items[9] ,case_items[10],case_items[11]) with open("../conf/cases.conf","w") as f: f.write( output ) return False @@ -286,27 +294,74 @@ def get_config(self): lines = f.readlines() for line in lines: p = line.split() - testcase_list.append( self.parse_benchmark_cases( p ) ) - except: + if len(p) != 0 and p!="\n": + testcase_list.append( self.parse_benchmark_cases( p ) ) + except: common.bash("cp %s %s" % (self.default_conf_path, self.conf_path)) with open(self.conf_path,"r") as f: lines = f.readlines() for line in lines: p = line.split() - testcase_list.append( self.parse_benchmark_cases( p ) ) + if len(p) != 0 and p!="\n": + testcase_list.append( self.parse_benchmark_cases( p ) ) return testcase_list - + def parse_benchmark_cases(self, testcase): p = testcase testcase_dict = { "benchmark_driver":p[0],"worker":p[1], "container_size":p[2], "iopattern":p[3], - "op_size":p[4], "object_size/QD":p[5], "rampup":p[6], "runtime":p[7], "device":p[8], "parameter":p[9] + "op_size":p[4], "object_size/QD":p[5], "rampup":p[6], "runtime":p[7], "device":p[8] } - if len(p) == 11: + + if len(p) == 12: + testcase_dict["parameter"] = p[9] testcase_dict["description"] = p[10] + testcase_dict["additional_option"] = p[11] else: - testcase_dict["description"] = "" + option_list = ['restart','redeploy'] + if len(p) == 9: + testcase_dict["parameter"] = "" + testcase_dict["description"] = "" + testcase_dict["additional_option"] = "" + elif len(p) == 10: + if self.check_parameter_style(p[9]): + testcase_dict["parameter"] = p[9] + testcase_dict["description"] = "" + testcase_dict["additional_option"] = "" + elif p[9] in option_list: + testcase_dict["parameter"] = "" + testcase_dict["description"] = "" + testcase_dict["additional_option"] = p[9] + else: + testcase_dict["parameter"] = "" + testcase_dict["description"] = p[9] + testcase_dict["additional_option"] = "" + + elif len(p) == 11: + if p[10] in option_list: + if self.check_parameter_style(p[9]): + testcase_dict["parameter"] = p[9] + testcase_dict["description"] = "" + testcase_dict["additional_option"] = p[10] + else: + testcase_dict["parameter"] = "" + testcase_dict["description"] = p[9] + testcase_dict["additional_option"] = p[10] + else: + testcase_dict["parameter"] = p[9] + testcase_dict["description"] = p[10] + testcase_dict["additional_option"] = "" + return testcase_dict + + def check_parameter_style(self,paras): + if paras != "": + for i in paras.split(','): + if len(i.split('=')) != 2: + return False + return True + else: + return False class ConfigHelper(): def _check_config( self, key, value, requirement=None): diff --git a/conf/default_value.conf b/conf/default_value.conf index 05070f8..109cf85 100644 --- a/conf/default_value.conf +++ b/conf/default_value.conf @@ -49,3 +49,4 @@ custom_script = "" disk_format = osd:journal aceph04 = /dev/sda1:/dev/sdc1 disable_tuning_check = false +distributed_data_process = false diff --git a/conf/description.py b/conf/description.py index 554e3c3..b84bd3b 100644 --- a/conf/description.py +++ b/conf/description.py @@ -27,7 +27,7 @@ def get_description_by_key(self,key): if key in Description.read_conf_to_dict().keys(): return Description.read_conf_to_dict()[key] else: - print "the key is not exists." + #print "the key is not exists." return "" class DefaultValue(object): diff --git a/conf/handler.py b/conf/handler.py index dc0b941..68a160f 100644 --- a/conf/handler.py +++ b/conf/handler.py @@ -125,7 +125,6 @@ def check_testcase(self): def check_engine(self, engine): required = OrderedDict() - required["Description"] = "width=10,depth=1,files=10000,threads=16,rdpct=65" if engine == "qemurbd": required["list_vclient"] = "vclient01,vclient02..." required["fio_capping"] = "false" @@ -209,6 +208,7 @@ def list_required_config(self): required_list["ceph_hard_config"]["osd_objectstore"] = "filestore" required_list["benchmark"] = OrderedDict() required_list["benchmark"]["disable_tuning_check"] = "false" + required_list["benchmark"]["distributed_data_process"] = "false" required_list["benchmark"]["tmp_dir"]="/opt/" required_list["benchmark"]["dest_dir"]="/mnt/data/" required_list["benchmark"]["cache_drop_level"]=1 @@ -216,7 +216,7 @@ def list_required_config(self): required_list["benchmark"]["collector"]="blktrace,strace,fatrace,lttng,perfcounter" required_list["benchmark"]["perfcounter_data_type"]="osd,filestore" required_list["benchmark"]["perfcounter_time_precision_level"]=6 - required_list["benchmark"]["Description"]="width=10,depth=1,files=10000,threads=16,rdpct=65" + #required_list["benchmark"]["Description"]="width=10,depth=1,files=10000,threads=16,rdpct=65" required_list["workflow"] = OrderedDict() required_list["workflow"]["workstages"] = ["deploy","benchmark"] diff --git a/conf/help.conf b/conf/help.conf index 5df1465..b440f85 100644 --- a/conf/help.conf +++ b/conf/help.conf @@ -47,3 +47,4 @@ cosbench_version = set to v0.4.c2 by default, when run a cosbench benchmark test custom_script = if benchmark_driver set to 'hook', user can add a '*.bash/*.sh' here, and CeTune will run this script at runtime. Ex: "bash hook.bash" disk_format = disk type and sequence of ${hostname}. disable_tuning_check = if true, cetune will not check ceph.conf before benchmark. +distributed_data_process = if true, cetune will distribute the data process work to all nodes and summarize in controller node. This option requires running controller_dependencies_install.py on all cetune workers. diff --git a/deploy/controller_dependencies_install.py b/deploy/controller_dependencies_install.py new file mode 100644 index 0000000..7f5e4c9 --- /dev/null +++ b/deploy/controller_dependencies_install.py @@ -0,0 +1,48 @@ +import os + +apt_pkg_list = ['python-pip','pdsh','unzip','zip','expect','sysstat','curl','openjdk-7-jre','haproxy','python-matplotlib','python-numpy','python-yaml','sqlite'] + +pip_pkg_list = ['ceph-deploy','pyyaml','argparse','markdown2'] +#update apt-get +os.system('sudo apt-get update') + +#install apt pkgs +apt_failed_install_list = [] +pip_failed_install_list = [] +for pkg in apt_pkg_list: + os.system('sudo apt-get install -y '+pkg) + #import pdb + output = os.popen('echo $?') + re_st = output.read().strip('\n') + if str(re_st) != "0": + apt_failed_install_list.append(pkg) + +#install pip pkgs +for pkg in pip_pkg_list: + os.system('sudo pip install '+pkg) + output = os.popen('echo $?') + re_st = output.read().strip('\n') + if str(re_st) != "0": + pip_failed_install_list.append(pkg) + +if len(apt_failed_install_list) != 0: + print 'APT Install Failed Pkgs:' + print '============================' + for i in apt_failed_install_list: + print i + print 'Solution:' + print '----------------------------' + print ' #sudo apt-get install -y package_name' + +if len(pip_failed_install_list) != 0: + print 'PIP Install Failed Pkgs:' + print '============================' + for i in apt_failed_install_list: + print i + print 'Solution:' + print '----------------------------' + print ' #sudo pip install package_name' + +if len(apt_failed_install_list) == 0 and len(pip_failed_install_list) == 0: + print "============================" + print "Successfully Installed !" diff --git a/deploy/mod/deploy.py b/deploy/mod/deploy.py index 11d9ddd..9d66690 100644 --- a/deploy/mod/deploy.py +++ b/deploy/mod/deploy.py @@ -341,9 +341,14 @@ def read_cephconf(self, request_type="json"): cephconf_dict["radosgw"] = [] tmp_dict = {} + cephconf = "" try: - with open("../conf/ceph_current_status", 'r') as f: - cephconf = f.readlines() + if not os.path.exists("../conf/ceph_current_status"): + with open("/etc/ceph/ceph.conf", 'r') as f: + cephconf = f.readlines() + else: + with open("../conf/ceph_current_status", 'r') as f: + cephconf = f.readlines() except: common.printout("WARNING", "Current Cluster ceph.conf file not exists under CeTune/conf/") return cephconf_dict @@ -764,12 +769,17 @@ def get_daemon_info_from_ceph_conf(self, daemon): "the daemon is not one of osd, mon or mds") sys.exit(1) + ceph_conf = "" try: - with open("../conf/ceph_current_status", 'r') as f: - ceph_conf = f.readlines() + if not os.path.exists("../conf/ceph_current_status"): + with open("/etc/ceph/ceph.conf", 'r') as f: + ceph_conf = f.readlines() + else: + with open("../conf/ceph_current_status", 'r') as f: + ceph_conf = f.readlines() except: common.printout("ERROR", - "Current Cluster ceph.conf file not exists under CeTune/conf/") + "Current Cluster ceph_current_status file not exists under CeTune/conf/") sys.exit(1) num = 0 @@ -828,6 +838,7 @@ def start_osd_created_by_ceph_disk(self): def start_osd(self): user = self.cluster["user"] osd_list = self.get_daemon_info_from_ceph_conf("osd") + print osd_list for osd in osd_list: osd_name = osd["daemon_name"] osd_host = osd["daemon_host"] @@ -844,3 +855,13 @@ def start_osd(self): 'exec %s"' % (lttng_prefix, cmd), option="console", except_returncode=1) common.printout("LOG","Started osd.%s daemon on %s" % (osd_name, osd_host)) + + def osd_perf_reset(self): + osd_list = self.get_daemon_info_from_ceph_conf("osd") + user = self.cluster["user"] + for osd in osd_list: + osd_name = osd["daemon_name"] + osd_host = osd["daemon_host"] + cmd = "ceph daemon osd.{0} perf reset all".format(osd_name) + common.pdsh(user, [osd_host],cmd) + common.printout("LOG","ceph daemon osd.{0} perf clean on {1}. ".format (osd_name, osd_host)) diff --git a/deploy/prepare-scripts/create_partition.sh b/deploy/prepare-scripts/create_partition.sh new file mode 100644 index 0000000..2e1fc20 --- /dev/null +++ b/deploy/prepare-scripts/create_partition.sh @@ -0,0 +1,119 @@ +#!/bin/bash +. ../../conf/common.sh +get_conf + +function list_device { + host=$1 + disk=$2 + echo ssh $host parted $disk p 2>/dev/null + ssh $host parted $disk p 2>/dev/null +} + +function do_partition_to_dev { + host=$1 + disk=$2 + partition_count=$3 + partition_size=$4 + no_dd=$5 + part_disk $host $disk $partition_count $partition_size $no_dd +} + +function part_disk { + host=$1 + device=$2 + part_count=$3 + part_size=$4 + no_dd=$5 + if [ "$no_dd" != "1" ]; then + echo "ssh $host dd if=/dev/zero of=$device bs=4M count=1 oflag=direct" + ssh $host "dd if=/dev/zero of=$device bs=4M count=1 oflag=direct" 2>/dev/null + echo "ssh $host parted $device mklabel gpt &>/dev/null" + ssh $host "parted $device mklabel gpt &>/dev/null" 2>/dev/null + start_pos="1" + end_pos=$part_size + for i in `seq 1 $part_count` + do + if [ -z $part_size ]; then + end_pos=`ssh $host "parted $device p" 2>/dev/null | grep "Disk $device" | awk '{print $3}'` + part_size="0" + fi + echo ssh $host "parted $device mkpart data $start_pos $end_pos" + ssh $host "parted $device mkpart data $start_pos $end_pos &>/dev/null" 2>/dev/null + start_pos=$end_pos + end_pos=$(( ${start_pos%%[[:alpha:]]*} + ${part_size%%[[:alpha:]]*} ))${part_size##*[0-9]} + done + else + start_pos=`ssh $host "parted $device p 2>/dev/null" 2>/dev/null | tac |sed -n '2p'| awk '{print $3}'` + end_pos=$(( ${start_pos%%[[:alpha:]]*} + ${part_size%%[[:alpha:]]*} ))${part_size##*[0-9]} + for i in `seq 1 $part_count` + do + echo ssh $host "parted $device mkpart journal $start_pos $end_pos" + ssh $host "parted $device mkpart journal $start_pos $end_pos &>/dev/null" 2>/dev/null + start_pos=$end_pos + end_pos=$(( ${start_pos%%[[:alpha:]]*} + ${part_size%%[[:alpha:]]*} ))${part_size##*[0-9]} + done + fi +} + +if [ $# -ne 5 ] && [ $# -ne 3 ] && [ $# -ne 5 ] && [ $# -ne 6 ] +then + echo $# + echo "Description:" + echo " This script is used to read conf/all.conf then print/format disk partition which will be used as osd and journal, pls always do -l before -w" + echo "Usage:" + echo " $0 [-l] hostname [disk,disk,...] or $0 [-w] hostname [disk,disk,...] partition_number partition_size [-a]" + exit +fi + +command=$1 +osd_host_list=$2 +disk=$3 +osd_partition_count=$4 +osd_partition_size=$5 +index=0 +for host in $osd_host_list +do + index=$(($index + 1)) + #disk=$(eval echo \$deploy_osd_server_$index) + osd_list=`echo $disk | sed 's/,/ /g'` + echo "osd devices:" + for item in $osd_list + do + if [[ $item == *"nvme"* ]]; then + osd_disk=`echo $item | sed 's/p$//g'` + else + osd_disk=`echo $item | sed 's/[0-9]\+$//g'` + fi + echo "/dev/"$osd_disk >> tmp.osd + done + case "$command" in + -l) + echo "============start list partition on host $host============" + ;; + -w) + echo "============start create partition on host $host============" + ;; + *) + echo "pls input -l or -w" + exit + ;; + esac + for osd_disk in `sort -u tmp.osd | sed 's/\n//g'` + do + case "$command" in + -l) + list_device $host $osd_disk + ;; + -w) + if [[ "$6" == "-a" ]] + then + do_partition_to_dev $host $osd_disk $osd_partition_count $osd_partition_size 1 + else + do_partition_to_dev $host $osd_disk $osd_partition_count $osd_partition_size + fi + ;; + esac + done + + rm tmp.osd +done diff --git a/deploy/run_deploy.py b/deploy/run_deploy.py index 547c2b0..90dee96 100644 --- a/deploy/run_deploy.py +++ b/deploy/run_deploy.py @@ -96,6 +96,9 @@ def main(args): if args.operation == "restart_rgw": mydeploy = deploy_rgw.Deploy_RGW() mydeploy.restart_rgw() + if args.operation == "osd_perf_reset": + mydeploy = deploy.Deploy() + mydeploy.osd_perf_reset() if __name__ == '__main__': import sys diff --git a/deploy/worker_dependencies_install.py b/deploy/worker_dependencies_install.py new file mode 100644 index 0000000..8d1c128 --- /dev/null +++ b/deploy/worker_dependencies_install.py @@ -0,0 +1,24 @@ +import os + +pkg_list = ['python-pip','unzip','sysstat','curl','openjdk-7-jre','haproxy'] + +#update apt-get +os.system('sudo apt-get update') + +#install pkg +failed_install_list = [] +for pkg in pkg_list: + os.system('sudo apt-get install -y '+pkg) + #import pdb + output = os.popen('echo $?') + re_st = output.read().strip('\n') + if str(re_st) != "0": + failed_install_list.append(pkg) +if len(failed_install_list) != 0: + print 'Install Failed Pkgs:' + print '============================' + for i in failed_install_list: + print i +else: + print "============================" + print "Successfully Installed !" diff --git a/pic/performance_portal_pics/overall_graph_cephfs_tuned.png b/static/pic/performance_portal_pics/overall_graph_cephfs_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_graph_cephfs_tuned.png rename to static/pic/performance_portal_pics/overall_graph_cephfs_tuned.png diff --git a/pic/performance_portal_pics/overall_graph_obj_tuned.png b/static/pic/performance_portal_pics/overall_graph_obj_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_graph_obj_tuned.png rename to static/pic/performance_portal_pics/overall_graph_obj_tuned.png diff --git a/pic/performance_portal_pics/overall_graph_obj_untuned.png b/static/pic/performance_portal_pics/overall_graph_obj_untuned.png similarity index 100% rename from pic/performance_portal_pics/overall_graph_obj_untuned.png rename to static/pic/performance_portal_pics/overall_graph_obj_untuned.png diff --git a/pic/performance_portal_pics/overall_graph_rbd_tuned.png b/static/pic/performance_portal_pics/overall_graph_rbd_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_graph_rbd_tuned.png rename to static/pic/performance_portal_pics/overall_graph_rbd_tuned.png diff --git a/pic/performance_portal_pics/overall_graph_rbd_untuned.png b/static/pic/performance_portal_pics/overall_graph_rbd_untuned.png similarity index 100% rename from pic/performance_portal_pics/overall_graph_rbd_untuned.png rename to static/pic/performance_portal_pics/overall_graph_rbd_untuned.png diff --git a/pic/performance_portal_pics/overall_table_cephfs_tuned.png b/static/pic/performance_portal_pics/overall_table_cephfs_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_table_cephfs_tuned.png rename to static/pic/performance_portal_pics/overall_table_cephfs_tuned.png diff --git a/pic/performance_portal_pics/overall_table_obj_tuned.png b/static/pic/performance_portal_pics/overall_table_obj_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_table_obj_tuned.png rename to static/pic/performance_portal_pics/overall_table_obj_tuned.png diff --git a/pic/performance_portal_pics/overall_table_obj_untuned.png b/static/pic/performance_portal_pics/overall_table_obj_untuned.png similarity index 100% rename from pic/performance_portal_pics/overall_table_obj_untuned.png rename to static/pic/performance_portal_pics/overall_table_obj_untuned.png diff --git a/pic/performance_portal_pics/overall_table_rbd_tuned.png b/static/pic/performance_portal_pics/overall_table_rbd_tuned.png similarity index 100% rename from pic/performance_portal_pics/overall_table_rbd_tuned.png rename to static/pic/performance_portal_pics/overall_table_rbd_tuned.png diff --git a/pic/performance_portal_pics/overall_table_rbd_untuned.png b/static/pic/performance_portal_pics/overall_table_rbd_untuned.png similarity index 100% rename from pic/performance_portal_pics/overall_table_rbd_untuned.png rename to static/pic/performance_portal_pics/overall_table_rbd_untuned.png diff --git a/pic/webui.png b/static/pic/webui.png similarity index 100% rename from pic/webui.png rename to static/pic/webui.png diff --git a/pic/webui_benchmark.png b/static/pic/webui_benchmark.png similarity index 100% rename from pic/webui_benchmark.png rename to static/pic/webui_benchmark.png diff --git a/pic/webui_deploy.png b/static/pic/webui_deploy.png similarity index 100% rename from pic/webui_deploy.png rename to static/pic/webui_deploy.png diff --git a/pic/webui_deploy_detail.png b/static/pic/webui_deploy_detail.png similarity index 100% rename from pic/webui_deploy_detail.png rename to static/pic/webui_deploy_detail.png diff --git a/pic/webui_result.png b/static/pic/webui_result.png similarity index 100% rename from pic/webui_result.png rename to static/pic/webui_result.png diff --git a/pic/webui_result_detail.png b/static/pic/webui_result_detail.png similarity index 100% rename from pic/webui_result_detail.png rename to static/pic/webui_result_detail.png diff --git a/pic/webui_result_detail2.png b/static/pic/webui_result_detail2.png similarity index 100% rename from pic/webui_result_detail2.png rename to static/pic/webui_result_detail2.png diff --git a/tuner/tuner.py b/tuner/tuner.py index e1af15a..3e102cd 100644 --- a/tuner/tuner.py +++ b/tuner/tuner.py @@ -3,7 +3,6 @@ sys.path.append(lib_path) from conf import * from deploy import * -from benchmarking import * import os, sys import time import pprint @@ -40,38 +39,6 @@ def default_all_conf(self): self.cluster = {} self.cluster["user"] = self.all_conf_data.get("user") - def run(self): - user = self.cluster["user"] - controller = self.cluster["head"] - osds = self.cluster["osds"] - pwd = os.path.abspath(os.path.join('..')) - if len(self.cluster["rgw"]) and self.cluster["rgw_enable"]=="true": - with_rgw = True - else: - with_rgw = False - for section in self.worksheet: - for work in self.worksheet[section]['workstages'].split(','): - if work == "deploy": - common.printout("LOG","Check ceph version, reinstall ceph if necessary") - self.apply_version(section) - self.apply_tuning(section, no_check=True) - common.printout("LOG","Start to redeploy ceph") - if with_rgw: - run_deploy.main(['--with_rgw','redeploy']) - else: - run_deploy.main(['redeploy']) - self.apply_tuning(section) - elif work == "benchmark": - if not common.check_ceph_running( user, controller ): - run_deploy.main(['restart']) - common.printout("LOG","start to run performance test") - if self.cluster["disable_tuning_check"] not in ["true", "True", "TRUE"]: - self.apply_tuning(section) - time.sleep(3) - run_cases.main(['--tuning', section]) - else: - common.printout("ERROR","Unknown tuner workstage %s" % work) - def handle_disk(self, option="get", param={'read_ahead_kb':2048, 'max_sectors_kb':512, 'scheduler':'deadline'}, fs_params=""): user = self.cluster["user"] osds = self.cluster["osds"] @@ -194,7 +161,10 @@ def dump_config(self): config = {} #get [system] config - config["disk"] = self.handle_disk(option="get") + try: + config["disk"] = self.handle_disk(option="get") + except: + pass #get [ceph version] #config['version'] = self.get_version() @@ -287,35 +257,17 @@ def apply_tuning(self, jobname, no_check = False): else: tmp_tuning_diff = ['global'] - if 'pool' in tmp_tuning_diff: - pool_exist = False - new_poolname = self.worksheet[jobname]['pool'].keys()[0] - if 'size' in self.worksheet[jobname]['pool'][new_poolname]: - replica_size = self.worksheet[jobname]['pool'][new_poolname]['size'] - else: - replica_size = 2 - if 'pg_num' not in self.worksheet[jobname]['pool'][new_poolname]: - new_pool_pg_num = 100 * self.cluster["osd_daemon_num"]/replica_size - else: - new_pool_pg_num = self.worksheet[jobname]['pool'][new_poolname]['pg_num'] - for cur_tuning_poolname in self.cur_tuning['pool'].keys(): - if cur_tuning_poolname != new_poolname: -# self.handle_pool(option = 'delete', param = {'name':cur_tuning_poolname}) - continue + if 'disk' in tmp_tuning_diff: + param = {} + for param_name, param_data in self.worksheet[jobname]['disk'].items(): + param[param_name] = param_data + try: + if param != {}: + self.handle_disk( option="set", param=param ) else: - if self.cur_tuning['pool'][cur_tuning_poolname]['pg_num'] != new_pool_pg_num: - self.handle_pool(option = 'delete', param = {'name':cur_tuning_poolname}) - else: - pool_exist = True - if not pool_exist: - self.handle_pool(option = 'create', param = {'name':new_poolname, 'pg_num':new_pool_pg_num}) - #after create pool, check pool param - latest_pool_config = self.get_pool_config() - for param in self.worksheet[jobname]['pool'][new_poolname]: - if param == 'pg_num' or param not in latest_pool_config[new_poolname]: - continue - if self.worksheet[jobname]['pool'][new_poolname][param] != latest_pool_config[new_poolname][param]: - self.handle_pool(option = 'set', param = {'name':new_poolname, param:self.worksheet[jobname]['pool'][new_poolname][param]}) + self.handle_disk( option="set" ) + except: + pass if 'global' in tmp_tuning_diff or 'osd' in tmp_tuning_diff or 'mon' in tmp_tuning_diff: if self.cluster["rgw_enable"]=="true" and len(self.cluster["rgw"]): with_rgw = True @@ -342,33 +294,42 @@ def apply_tuning(self, jobname, no_check = False): run_deploy.main(['--with_rgw','restart']) else: run_deploy.main(['restart']) - if 'disk' in tmp_tuning_diff: - param = {} - for param_name, param_data in self.worksheet[jobname]['disk'].items(): - param[param_name] = param_data - if param != {}: - self.handle_disk( option="set", param=param ) + if 'pool' in tmp_tuning_diff: + pool_exist = False + new_poolname = self.worksheet[jobname]['pool'].keys()[0] + if 'size' in self.worksheet[jobname]['pool'][new_poolname]: + replica_size = int(self.worksheet[jobname]['pool'][new_poolname]['size']) else: - self.handle_disk( option="set" ) + replica_size = 2 + if 'pg_num' not in self.worksheet[jobname]['pool'][new_poolname]: + new_pool_pg_num = 100 * self.cluster["osd_daemon_num"]/replica_size + else: + new_pool_pg_num = self.worksheet[jobname]['pool'][new_poolname]['pg_num'] + for cur_tuning_poolname in self.cur_tuning['pool'].keys(): + if cur_tuning_poolname != new_poolname: +# self.handle_pool(option = 'delete', param = {'name':cur_tuning_poolname}) + continue + else: + if self.cur_tuning['pool'][cur_tuning_poolname]['pg_num'] != new_pool_pg_num: + self.handle_pool(option = 'delete', param = {'name':cur_tuning_poolname}) + else: + pool_exist = True + if not pool_exist: + self.handle_pool(option = 'create', param = {'name':new_poolname, 'pg_num':new_pool_pg_num}) + #after create pool, check pool param + latest_pool_config = self.get_pool_config() + for param in self.worksheet[jobname]['pool'][new_poolname]: + if param == 'pg_num' or param not in latest_pool_config[new_poolname]: + continue + if self.worksheet[jobname]['pool'][new_poolname][param] != latest_pool_config[new_poolname][param]: + self.handle_pool(option = 'set', param = {'name':new_poolname, param:self.worksheet[jobname]['pool'][new_poolname][param]}) if no_check: return - #wait ceph health to be OK - waitcount = 0 - try: - while not self.check_health() and waitcount < 300: - common.printout("WARNING","Applied tuning, waiting ceph to be healthy") - time.sleep(3) - waitcount += 3 - except: - common.printout("WARNING","Caught KeyboardInterrupt, exit") - sys.exit() - if waitcount < 300: - common.printout("LOG","Tuning has applied to ceph cluster, ceph is Healthy now") - else: - common.printout("ERROR","ceph is unHealthy after 300sec waiting, please fix the issue manually") - sys.exit() + user = self.cluster["user"] + controller = self.cluster["head"] + common.wait_ceph_to_health( user, controller ) def handle_pool(self, option="set", param = {}): user = self.cluster["user"] @@ -398,37 +359,28 @@ def handle_pool(self, option="set", param = {}): common.printout("LOG","delete ceph pool %s" % pool) common.pdsh(user, [controller], "ceph osd pool delete %s %s --yes-i-really-really-mean-it" % (pool, pool), option="check_return") - def check_health(self): - user = self.cluster["user"] - controller = self.cluster["head"] - check_count = 0 - stdout, stderr = common.pdsh(user, [controller], 'ceph health', option="check_return") - if "HEALTH_OK" in stdout: - return True - else: - return False - def main(args): - parser = argparse.ArgumentParser(description='tuner') - parser.add_argument( - '--by_thread', + print args + tuner_parser = argparse.ArgumentParser(description='Tuner.') + tuner_parser.add_argument( + 'operation', + ) + tuner_parser.add_argument( + '--section', + ) + tuner_parser.add_argument( + '--no_check', default = False, action = 'store_true' - ) - args = parser.parse_args(args) + ) + args = tuner_parser.parse_args(args) tuner = Tuner() - if args.by_thread: - print "tuner by thread" - new_thread = threading.Thread(target=tuner.run, args=()) - new_thread.daemon = True - new_thread.start() - return new_thread - else: - tuner.run() - return None + if args.operation == "apply_tuning": + tuner.apply_tuning( args.section, args.no_check ) + if args.operation == "apply_version": + tuner.apply_version( args.section ) if __name__ == '__main__': - print "enter tuner" - tuner = Tuner() - tuner.run() + import sys + main( sys.argv[1:] ) #tuner.apply_tuning('testjob1') diff --git a/visualizer/create_DB.py b/visualizer/create_DB.py index b3cd005..7ab6fcc 100644 --- a/visualizer/create_DB.py +++ b/visualizer/create_DB.py @@ -25,6 +25,7 @@ def createTB(self,dbpath): iops float not null, bw float not null, latency float not null, + latency_99 float not null, sniops float not null, snbw float not null, snlatency float not null @@ -43,8 +44,13 @@ def insert_to_TB(self,data,dbpath): rowdata[2] =='None' if rowdata[4] == '': rowdata[4] = 'None' - sqlstr = "insert into tb_report (runid,runid_tr,timestamp,status,description,opsize,optype,qd,driver,snnumber,cnnumber,worker,runtime,iops,bw,latency,sniops,snbw,snlatency) values ("+rowdata[1]+",'"+rowdata[0]+"','"+rowdata[2]+"','"+rowdata[3]+"','"+rowdata[4]+"','"+rowdata[5]+"','"+rowdata[6]+"','"+rowdata[7]+"','"+rowdata[8]+"',"+rowdata[9]+","+rowdata[10]+","+rowdata[11]+","+rowdata[12]+",'"+rowdata[13]+"','"+rowdata[14]+"','"+rowdata[15]+"','"+rowdata[16]+"','"+rowdata[17]+"','"+rowdata[18]+"')" + if len(rowdata) == 19: + sqlstr = "insert into tb_report (runid,runid_tr,timestamp,status,description,opsize,optype,qd,driver,snnumber,cnnumber,worker,runtime,iops,bw,latency,latency_99,sniops,snbw,snlatency) values ("+rowdata[1]+",'"+rowdata[0]+"','"+rowdata[2]+"','"+rowdata[3]+"','"+rowdata[4]+"','"+rowdata[5]+"','"+rowdata[6]+"','"+rowdata[7]+"','"+rowdata[8]+"',"+rowdata[9]+","+rowdata[10]+","+rowdata[11]+","+rowdata[12]+",'"+rowdata[13]+"','"+rowdata[14]+"','"+rowdata[15]+"','0.00','"+rowdata[16]+"','"+rowdata[17]+"','"+rowdata[18]+"')" + else: + sqlstr = "insert into tb_report (runid,runid_tr,timestamp,status,description,opsize,optype,qd,driver,snnumber,cnnumber,worker,runtime,iops,bw,latency,latency_99,sniops,snbw,snlatency) values ("+rowdata[1]+",'"+rowdata[0]+"','"+rowdata[2]+"','"+rowdata[3]+"','"+rowdata[4]+"','"+rowdata[5]+"','"+rowdata[6]+"','"+rowdata[7]+"','"+rowdata[8]+"',"+rowdata[9]+","+rowdata[10]+","+rowdata[11]+","+rowdata[12]+",'"+rowdata[13]+"','"+rowdata[14]+"','"+rowdata[15]+"','"+rowdata[16]+"','"+rowdata[17]+"','"+rowdata[18]+"','"+rowdata[19]+"')" + #sqlstr = "insert into tb_report (runid,runid_tr,utatus,description,opsize,optype,qd,driver,snnumber,cnnumber,worker,runtime,iops,bw,latency,sniops,snbw,snlatency) values (%d,'%s','%s','%s','%s','%s','%s','%s',%d,%d,%d,%d,%f,%f,%f,%f,%f,%f)"%(rowdata[1],rowdata[0],rowdata[2],rowdata[3],rowdata[4],rowdata[5],rowdata[6],rowdata[7],rowdata[8],rowdata[9],rowdata[10],rowdata[11],rowdata[12],rowdata[13],rowdata[14],rowdata[15],rowdata[16],rowdata[17]) + print sqlstr conn.execute(sqlstr) conn.commit() print "Add data to TB successfully." diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 7a62ecf..bebb8e2 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -177,13 +177,19 @@ def update_report_list_db(self,tr_id,new_description): def check_DB_case_list(self,re_dir,dbpath): if os.path.exists(dbpath): output = os.popen("ls "+re_dir) - local_list = output.readlines() - local_case_list = [] - for i in local_list: - if i != 'cetune_report.db': - local_case_list.append(i) + li = output.readlines() + local_list = [] + for i in li: + if os.path.exists(os.path.join(re_dir,i.strip('\n'),i.strip('\n')+'.html')): + local_list.append(i.strip('\n')) + #local_case_list = [] + #for i in local_list: + # if i != 'cetune_report.db': + # local_case_list.append(i) DB_list = database.get_runid_list(dbpath) - if local_case_list == DB_list: + local_list.sort() + DB_list.sort() + if local_list == DB_list: return True else: return False @@ -194,20 +200,23 @@ def generate_history_view(self, remote_host="127.0.0.1", remote_dir="/mnt/data/" common.printout("LOG","Generating history view") dbpath = os.path.join(self.db_path,"cetune_report.db") if not self.check_DB_case_list(self.db_path,dbpath): - stdout, stderr = common.pdsh(user, [remote_host], "find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir, option="check_return") - res = common.format_pdsh_return(stdout) - if remote_host not in res: - common.printout("ERROR","Generating history view failed") - return False + #stdout, stderr = common.pdsh(user, [remote_host], "find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir, option="check_return") + #res = common.format_pdsh_return(stdout) + #if remote_host not in res: + # common.printout("ERROR","Generating history view failed") + # return False # some modification in greped trs + stdout = common.bash("find %s -name '*.html' | grep -v 'cetune_history'|sort -u | while read file;do session=`echo $file | awk -F/ {'print $(NF-1)'}`; awk -v session=\"$session\" 'BEGIN{find=0;}{if(match($1,\"tbody\")&&find==2){find=0;}if(find==2){if(match($1,\"\");else print ;};if(match($1,\"div\")&&match($2,\"summary\"))find=1;if(match($1,\"tbody\")&&find==1){find+=1}}' $file; done" % remote_dir) + res_tmp = stdout; formated_report = {} - report_lines = re.findall('()',res[remote_host],re.S) + report_lines = re.findall('()',res_tmp,re.S) for line in report_lines: tr_start = re.search('()', line, re.S).group(1) data = re.findall('(.*?)', line, re.S) - runid = int(data[0]) - if len(data) < 17: + #runid = int(data[0]) + runid = re.findall('id=(.*?)>', tr_start, re.S)[0] + if len(data) < 18: data.insert(2, "") formated_report[runid] = tr_start for block in data: @@ -219,6 +228,8 @@ def generate_history_view(self, remote_host="127.0.0.1", remote_dir="/mnt/data/" database.createTB(dbpath) rows = self.dataparse(formated_report) runid_list = [] + while [] in rows: + rows.remove([]) for i in rows: runid_list.append(i[0]) if not database.check_case_exist(i[0],dbpath): @@ -238,7 +249,7 @@ def generate_history_view(self, remote_host="127.0.0.1", remote_dir="/mnt/data/" output.extend( self.getSummaryTitle() ) #output.append(" ") #output.append(" ") - #output.append(res[remote_host]) + #output.append(res_tmp) for runid in sorted(lines.keys()): output.append(lines[runid]) #output.append(" ") @@ -281,9 +292,10 @@ def getSummaryTitle(self): output.append(" IOPS") output.append(" BW(MB/s)") output.append(" Latency(ms)") - output.append(" SN_IOPS") - output.append(" SN_BW(MB/s)") - output.append(" SN_Latency(ms)") + output.append(" 99.99% Latency(ms)") + output.append(" SN_IOPS") + output.append(" SN_BW(MB/s)") + output.append(" SN_Latency(ms)") output.append(" ") return output diff --git a/webui/account.conf b/webui/account.conf new file mode 100644 index 0000000..24304d0 --- /dev/null +++ b/webui/account.conf @@ -0,0 +1,3 @@ +[account] +admin = 123456 +root = 123456 diff --git a/webui/login.py b/webui/login.py new file mode 100644 index 0000000..03b9e4b --- /dev/null +++ b/webui/login.py @@ -0,0 +1,35 @@ +#/usr/bin/python +import ConfigParser +import os +import collections + +class UserClass(object): + @classmethod + def read_conf_to_dict(self): + self.cf = ConfigParser.ConfigParser() + self.dict = collections.OrderedDict() + if os.path.exists("account.conf"): + self.cf.read("account.conf") + self.data = self.cf.items("account") + self.data = sorted(self.data) + for i in range(len(self.data)): + self.dict[self.data[i][0]]=self.data[i][1] + else: + print "ERROR:account.conf not exists." + return self.dict + + @classmethod + def get_all_account(self): + return UserClass.read_conf_to_dict() + + @classmethod + def check_account(self,key=[' ',' ']): + result = 'false' + if key[0] in UserClass.read_conf_to_dict().keys(): + if UserClass.read_conf_to_dict()[key[0]] == key[1]: + result = 'true' + return result + return result + else: + print "ERROR:user not exists." + return result diff --git a/webui/static/css/Style.css b/webui/static/css/Style.css index 9cde7b5..e2c5b1c 100644 --- a/webui/static/css/Style.css +++ b/webui/static/css/Style.css @@ -31,8 +31,9 @@ body{ margin:0px; padding:0px;} /***************************************************************************************************************/ #div_top_id{ position:fixed; top:25px; z-index:100; width:100%; height:55px; background-color:#337ab7; border-bottom:1px solid #2e6da4; box-shadow:2px 2px 3px #aaa;} -#div_top_title_id{ float:left; width:50%; height:50px;} -#div_top_status_id{ float:right; width:45%; height:50px;} +#div_top_title_id{ float:left; width:45%; height:50px;} +#div_top_status_id{ float:left; width:45%; height:50px;} +#div_user_status_id{ float:right; width:10%;font-size:14px; height:50px;color:#FFF;font-weight:bold;margin-top:5px;} #div_top_id h1{ margin:0; font-family:Arial, Helvetica, sans-serif; font-style:italic;font-weight:500; line-height:1.1; font-size: 20px; color: #FFF; diff --git a/webui/static/js/Common.js b/webui/static/js/Common.js index b335123..9fdbc8f 100644 --- a/webui/static/js/Common.js +++ b/webui/static/js/Common.js @@ -171,6 +171,11 @@ function CreateTableHTML_Benchmark(jsonObj){ tableHtml += "Description"; tableHtml += ""; + tableHtml += ""; + tableHtml += "additional_option"; + tableHtml += ""; + + tableHtml += "" tableHtml += ""; @@ -237,12 +242,53 @@ function CreateTableHTML_Benchmark(jsonObj){ tableHtml += ""; tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + if(val.additional_option == "restart"){ + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + } + if(val.additional_option == "resetPerf"){ + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + tableHtml += ""; + } + if(val.additional_option == ""){ + tableHtml += "" + tableHtml += ""; + tableHtml += ""; }); tableHtml += ""; - tableHtml += "" + tableHtml += ""; + tableHtml += "
  • If case start with '#',workflow will not running this cases.
    • "; return tableHtml; } @@ -309,6 +355,12 @@ function Cancel_Apply(rowNum,value){ otd.innerHTML =" "; } //------------------------------------------------------------------------------------------------------ +//radio button click opertion +function Additional_option_change(){ + Submit_Benchmark(); + CheckTableDataError(); +} + //label click opertion function Label_benchmark_Click(rowNum , colNum , value){ olabel = document.getElementById("label_benchmark_id_" + rowNum + "_" + colNum); @@ -359,10 +411,12 @@ function Del(tableType){ } }); } -//------------------------- +//------------------------ else if(tableType =="benchmark"){ + var rowNum; $(".checkbox_benchmark_class").each(function(index,value){ if($(this).is(':checked')){ + rowNum = index; $(this).parent().parent().remove(); } }); @@ -425,7 +479,13 @@ function DeleteModal_OK(type){ } function Select_Value(){ - var _example_list = [['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','librados_4MB_write'],['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','librados_4MB_write'],['0(use 0 to initialize container) and 160(as worker)','r(1,100)','write, read','128KB','r(1,100)','100','400','librados_4MB_write'],['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','librados_4MB_write'],[' ',' ',' ',' ',' ',' ','400','librados_4MB_write'],['50','30g','seqwrite,seqread,randwrite,randread,readwrite,randrw','8k','32','30','120','librados_4MB_write']] + var _example_list = [ + ['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','rbd','librados_4MB_write','restart,redeploy,resetPerf'], + ['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','rbd','librados_4MB_write','restart,redeploy,resetPerf'], + ['0(use 0 to initialize container) and 160(as worker)','r(1,100)','write, read','128KB','r(1,100)','100','400','','librados_4MB_write','restart,redeploy,resetPerf'], + ['140','10g','seqwrite,seqread,randwrite,randread,readwrite,randrw','4k','64','100','400','','librados_4MB_write','restart,redeploy,resetPerf'], + [' ',' ',' ',' ',' ',' ','400','','librados_4MB_write','restart,redeploy,resetPerf'], + ['50','30g','seqwrite,seqread,randwrite,randread,readwrite,randrw','8k','32','30','120','width=10,depth=1,files=10000,threads=16,rdpct=65','librados_4MB_write','restart,redeploy,resetPerf']] var select_value = document.getElementById("recipient_benchmark_engine"); var item = 0 if(select_value.value == "qemurbd"){ @@ -465,10 +525,14 @@ function Select_Value(){ document.getElementById('recipient_work_depth').readOnly=readonly; document.getElementById('recipient_ramup_time').readOnly=readonly; if(select_value.value == "vdbench"){ - width=10,depth=1,files=10000,threads=16,rdpct=65 document.getElementById("recipient_parameter").value = "width=10,depth=1,files=10000,threads=16,rdpct=65"; document.getElementById('recipient_parameter').readOnly=false; } + if(select_value.value == "fiorbd" || select_value.value == "qemurbd"){ + document.getElementById("recipient_parameter").value = "rbd"; + document.getElementById('recipient_parameter').readOnly=false; + } + document.getElementById("additional_option_select").value = 'restart'; var worker = document.getElementById("worker"); worker.innerHTML ="example: "+ _example_list[item][0] var container_size = document.getElementById("container_size"); @@ -483,8 +547,12 @@ function Select_Value(){ rampup.innerHTML = "example: "+ _example_list[item][5] var runtime = document.getElementById("runtime"); runtime.innerHTML = "example: "+ _example_list[item][6] + var parameter = document.getElementById("Parameter"); + parameter.innerHTML = "example: "+ _example_list[item][7] var Tag_Description = document.getElementById("Tag_Description"); - Tag_Description.innerHTML = "example: "+ _example_list[item][7] + Tag_Description.innerHTML = "example: "+ _example_list[item][8] + var additional_option = document.getElementById("additional_option_label"); + additional_option.innerHTML = "example: "+ _example_list[item][9] } function ConfigurationModal_OK(key, value, dsc){ @@ -544,7 +612,7 @@ function Append_Row_to_Configuration(params){ $("table.table_class tr:nth-child(odd)").addClass("altrow"); } -function CheckInput(key,value){ +function CheckInput(key="",value=""){ var chk_result = 'true'; if(value !== ""){ if(key == "parameter"){ @@ -582,6 +650,7 @@ function BenchMarkModel_OK(){ var parameter = document.getElementById("recipient_parameter").value; //var desc = $("#recipient-desc").val(); var desc = document.getElementById("recipient_desc").value; + var additional_option = document.getElementById("additional_option_select").value; if(benchmark_driver == "qemurbd") device = "/dev/vdb" if(benchmark_driver == "fiorbd") @@ -594,7 +663,8 @@ function BenchMarkModel_OK(){ device = "/dev/vdb" if(benchmark_driver == "hook") device = "hook" - if(CheckInput('parameter',parameter) == "true"){ + //if(CheckInput('parameter',parameter) == "true"){ + if(CheckInput() == "true"){ if(benchmark_driver == "" || worker== "" ||container_size == "" || iopattern == "" || op_size == "" || object_size == "" || rampup == "" || runtime == "" || device == "" ){ @@ -644,6 +714,46 @@ function BenchMarkModel_OK(){ html +=""; html +=""; + html += ""; + html += ""; + if(additional_option == "restart"){ + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + } + if(additional_option == "resetPerf"){ + html += ""; + html += ""; + html += ""; + html += ""; + html += ""; + } + if(additional_option == ""){ + html += "" + html +=""; + html += ""; $("#table_benchmark_id").append(html); @@ -670,6 +780,9 @@ function Submit_Benchmark(){ device = $(this).parent().parent().children().eq(9).children("label").text(); parameter = $(this).parent().parent().children().eq(10).children("label").text(); desc = $(this).parent().parent().children().eq(11).children("label").text(); + var additional = ''; + var rowid = $(this).parent().parent().children().eq(12).children().eq(0).text(); + additional = document.getElementById("additional_option_dropdown_"+rowid).value; var data ={}; data.benchmark_driver = benchmark_driver; @@ -683,7 +796,8 @@ function Submit_Benchmark(){ data.device = device; data.parameter = parameter; data.desc = desc; - + if(additional == "null"){data.additional_option="";} + else{data.additional_option = additional;} table_data.push(data); }); diff --git a/webui/static/js/Script.js b/webui/static/js/Script.js index 97a26d3..658f8e2 100644 --- a/webui/static/js/Script.js +++ b/webui/static/js/Script.js @@ -237,7 +237,7 @@ function Report_Timer(init){ appendHtml +=""+ str +""; - appendHtml +=""; + appendHtml +=""; appendHtml +=""; @@ -345,7 +345,7 @@ $(document).ready(function(){ case "menu_Configuration_id": clearTimer(timer_Console); clearTimer(timer_Report); - RunStatus_Timer(); + //RunStatus_Timer(); timer_RunStatus = setInterval(RunStatus_Timer,interval_RunStatus); break; @@ -432,9 +432,9 @@ $(document).ready(function(){ //traverse the sub menu li, check Configuation Data is true---------------------------------- - CheckTableDataError(); + //CheckTableDataError(); - ExecutvieCheckSync(); + //ExecutvieCheckSync(); //Executive if(CheckIsExecutive() == "true"){ @@ -584,7 +584,7 @@ function Init(){ //(4)decide server is runing,start the timer; - RunStatus_Timer(); + //RunStatus_Timer(); timer_RunStatus = setInterval(RunStatus_Timer,interval_RunStatus); } @@ -640,12 +640,17 @@ function loading(){ function DisplayConfiguationDataTable(request_type){ //(1) get data for json obj var jsonObj_Config = GetConfigurationData(request_type); - - var address_Benchmark = "../configuration/get_group?request_type=testcase"; - var jsonObj_Benchmark = GetDataByAjax(address_Benchmark); - + + var jsonObj_Benchmark; + if(request_type == "benchmark"){ + var address_Benchmark = "../configuration/get_group?request_type=testcase"; + jsonObj_Benchmark = GetDataByAjax(address_Benchmark); + //var jsonObj_Benchmark = GetDataByAjax(address_Benchmark); + console.log("jsonObj_Bnechmark"); + } //(2) display table CreateDataTableForConfiguration(jsonObj_Config , jsonObj_Benchmark ,request_type); + console.log("CreateDataTableForConfiguration"); } //get data by ajax(post) diff --git a/webui/static/login/css/style.css b/webui/static/login/css/style.css new file mode 100644 index 0000000..7d4055f --- /dev/null +++ b/webui/static/login/css/style.css @@ -0,0 +1,220 @@ +/* + * + * Template Name: Fullscreen Login + * Description: Login Template with Fullscreen Background Slideshow + * Author: Anli Zaimi + * Author URI: http://azmind.com + * + */ + + +body { + background: #f8f8f8; + background-image:url(../img/backimg.png); + text-align: center; + font-size:14px; + color: #fff; + font-family:Microsoft YaHei,Segoe UI,Tahoma,Arial,Verdana,sans-serif; +} + +#loading { + margin:0 auto; + display:none; +} +/*#loading img{ + border-radius: 50%; + width: 500px; + height: 500px; + overflow: hidden; + position:relative; +}*/ +#loading img{ + position:absolute; + content:''; + width:450px; + height:380px; + top:0;left:0; + border-radius:10%; + box-shadow:0 0 30px 10px rgba(255,255,255,.7) inset; +} + +.page-container { + width:350px; + height:auto; + margin: 120px auto 0 auto; +} + +h1 { + font-size: 30px; + font-weight: 700; + text-shadow: 0 1px 4px rgba(0,0,0,.2); +} + +form { + position: relative; + width: 350px; + margin: 15px auto 0 auto; + text-align: center; +} + +#username,#password { + width: 270px; + height: 42px; + margin-top: 25px; + padding: 0 15px; + background: #2d2d2d; /* browsers that don't support rgba */ + background: rgba(45,45,45,.15); + -moz-border-radius: 6px; + -webkit-border-radius: 6px; + border-radius: 6px; + border: 1px solid #3d3d3d; /* browsers that don't support rgba */ + border: 1px solid rgba(255,255,255,.15); + -moz-box-shadow: 0 2px 3px 0 rgba(0,0,0,.1) inset; + -webkit-box-shadow: 0 2px 3px 0 rgba(0,0,0,.1) inset; + box-shadow: 0 2px 3px 0 rgba(0,0,0,.1) inset; + font-family: 'PT Sans', Helvetica, Arial, sans-serif; + font-size: 14px; + color: #fff; + text-shadow: 0 1px 2px rgba(0,0,0,.1); + -o-transition: all .2s; + -moz-transition: all .2s; + -webkit-transition: all .2s; + -ms-transition: all .2s; +} + +#username,#password :-moz-placeholder { color: #fff; } +#username,#password :-ms-input-placeholder { color: #fff; } +#username,#password ::-webkit-input-placeholder { color: #fff; } + +#username,#password :focus { + outline: none; + -moz-box-shadow: + 0 2px 3px 0 rgba(0,0,0,.1) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + -webkit-box-shadow: + 0 2px 3px 0 rgba(0,0,0,.1) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + box-shadow: + 0 2px 3px 0 rgba(0,0,0,.1) inset, + 0 2px 7px 0 rgba(0,0,0,.2); +} + +button { + cursor: pointer; + width: 300px; + height: 44px; + margin-top: 25px; + padding: 0; + background: #ef4300; + -moz-border-radius: 6px; + -webkit-border-radius: 6px; + border-radius: 6px; + border: 1px solid #ff730e; + -moz-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.25) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + -webkit-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.25) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + box-shadow: + 0 15px 30px 0 rgba(255,255,255,.25) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + font-family: 'PT Sans', Helvetica, Arial, sans-serif; + font-size: 14px; + font-weight: 700; + color: #fff; + text-shadow: 0 1px 2px rgba(0,0,0,.1); + -o-transition: all .2s; + -moz-transition: all .2s; + -webkit-transition: all .2s; + -ms-transition: all .2s; +} + +button:hover { + -moz-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.15) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + -webkit-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.15) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + box-shadow: + 0 15px 30px 0 rgba(255,255,255,.15) inset, + 0 2px 7px 0 rgba(0,0,0,.2); +} + +button:active { + -moz-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.15) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + -webkit-box-shadow: + 0 15px 30px 0 rgba(255,255,255,.15) inset, + 0 2px 7px 0 rgba(0,0,0,.2); + box-shadow: + 0 5px 8px 0 rgba(0,0,0,.1) inset, + 0 1px 4px 0 rgba(0,0,0,.1); + + border: 0px solid #ef4300; +} + +.error { + display: none; + position: absolute; + top: 27px; + right: -55px; + width: 40px; + height: 40px; + background: #2d2d2d; /* browsers that don't support rgba */ + background: rgba(45,45,45,.25); + -moz-border-radius: 8px; + -webkit-border-radius: 8px; + border-radius: 8px; +} + +.error span { + display: inline-block; + margin-left: 2px; + font-size: 40px; + font-weight: 700; + line-height: 40px; + text-shadow: 0 1px 2px rgba(0,0,0,.1); + -o-transform: rotate(45deg); + -moz-transform: rotate(45deg); + -webkit-transform: rotate(45deg); + -ms-transform: rotate(45deg); + +} + +.connect { + width: 305px; + margin: 35px auto 0 auto; + font-size: 18px; + font-weight: 700; + text-shadow: 0 1px 3px rgba(0,0,0,.2); +} + +.connect a { + display: inline-block; + width: 32px; + height: 35px; + margin-top: 15px; + -o-transition: all .2s; + -moz-transition: all .2s; + -webkit-transition: all .2s; + -ms-transition: all .2s; +} + +.connect a.facebook { background: url(../img/facebook.png) center center no-repeat; } +.connect a.twitter { background: url(../img/twitter.png) center center no-repeat; } + +.connect a:hover { background-position: center bottom; } + +/*－－－增加部份的CSS－－－*/ +#username,#password :{font-family:Microsoft YaHei,Segoe UI,Tahoma,Arial,Verdana,sans-serif; text-decoration:none; width:80%;} +form,button{font-family:Microsoft YaHei,Segoe UI,Tahoma,Arial,Verdana,sans-serif; text-decoration:none; width:80%;} +button.submit_button{ font-size:24px; letter-spacing:15px;} +#username.Captcha{ width:130px; float:left;} +#password.Captcha{ width:130px; float:left;} + + + + diff --git a/webui/static/login/img/backimg.png b/webui/static/login/img/backimg.png new file mode 100644 index 0000000..741a992 Binary files /dev/null and b/webui/static/login/img/backimg.png differ diff --git a/webui/static/login/img/userpic1.png b/webui/static/login/img/userpic1.png new file mode 100644 index 0000000..3444dae Binary files /dev/null and b/webui/static/login/img/userpic1.png differ diff --git a/webui/static/index.html b/webui/templates/index.html similarity index 87% rename from webui/static/index.html rename to webui/templates/index.html index fb3dc1f..1a95320 100644 --- a/webui/static/index.html +++ b/webui/templates/index.html @@ -12,27 +12,27 @@ CeTune - - - - - - - + + + + + + + - - + + - - - + + + - - - - - + + + + + @@ -53,10 +53,16 @@

    CeTune -- Ceph tuning and profiling

    + +

    CeTune Status: idle Ceph Status: None

    +
    @@ -74,19 +80,19 @@
    @@ -364,9 +370,9 @@
    @@ -398,7 +415,7 @@