From fe0be8dc5958ea084a1a7e6eeb88a977ac6a6d4c Mon Sep 17 00:00:00 2001 From: ShaoShianXA Date: Fri, 16 Dec 2016 10:24:11 +0800 Subject: [PATCH] save the interrupt to the node. Signed-off-by: shaoshian <1196753438@qq.com> Signed-off-by: shaoshian --- analyzer/analyzer.py | 59 ++++++++++++++++++++++------------- analyzer/analyzer_remote.py | 47 +++++++++++++++++----------- benchmarking/mod/benchmark.py | 4 +++ 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/analyzer/analyzer.py b/analyzer/analyzer.py index a359949..aa37257 100644 --- a/analyzer/analyzer.py +++ b/analyzer/analyzer.py @@ -16,6 +16,7 @@ import copy import getpass from multiprocessing import Process +import csv pp = pprint.PrettyPrinter(indent=4) class Analyzer: @@ -104,7 +105,6 @@ def print_remote_log(self,node_log,node): except Exception as e: common.printout("WARNING","print_remote_log failed") common.printout("WARNING",str(e)) - def process_data(self): case_type = re.findall('\d\-\S+', self.cluster["dest_dir"])[0].split('-')[2] @@ -138,7 +138,6 @@ def process_data(self): common.scp(self.cluster["user"],node,remote_file2,self.cluster["tmp_dir"]) common.scp(self.cluster["user"],node,remote_file3,self.cluster["tmp_dir"]) common.scp(self.cluster["user"],node,remote_file4,self.cluster["tmp_dir"]) - try: common.pdsh(self.cluster["user"],[node],"echo \"\" > " + self.cluster["tmp_dir"] +node+"-cetune_console.log") except: @@ -158,6 +157,7 @@ def process_data(self): else: common.rscp(self.cluster["user"],node,self.workpath,os.path.join(self.cluster["tmp_dir"],node+"-system.json")) common.rscp(self.cluster["user"],node,self.workpath,os.path.join(self.cluster["tmp_dir"],node+"-workload.json")) + common.rscp(self.cluster["user"],node,self.cluster["dest_dir"].replace('raw','conf'),os.path.join(self.cluster["tmp_dir"],node+"_interrupt.csv")) self.print_remote_log(log_line,node) all_node.remove((proc,node)) if not len(all_node): @@ -196,7 +196,6 @@ def process_data(self): workload = self._process_remote_data(workload_file) self.result["vclient"][dir_name]=system self.result["workload"].update(workload) - #-------------------remote end-------------------------- else: for dir_name in os.listdir(dest_dir): @@ -495,8 +494,8 @@ def _process_data(self, node_name): result.update(res) if '_interrupts_end.txt' in dir_name: if os.path.exists("%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start'))): - interrupt_start = "%s/%s/%s" % (dest_dir, node_name, dir_name) - interrupt_end = "%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start')) + interrupt_end = "%s/%s/%s" % (dest_dir, node_name, dir_name) + interrupt_start = "%s/%s/%s" % (dest_dir, node_name, dir_name.replace('end','start')) self.interrupt_diff(dest_dir,node_name,interrupt_start,interrupt_end) if '_process_log.txt' in dir_name: res = self.process_log_data( "%s/%s/%s" % (dest_dir, node_name, dir_name) ) @@ -522,8 +521,9 @@ def process_smartinfo_data(self, path): def interrupt_diff(self,dest_dir,node_name,s_path,e_path): s_p = s_path e_p = e_path - result_name = node_name+'_interrupt.txt' - result_path = os.path.join(dest_dir.replace('raw','conf'),result_name) + result_name = node_name+'_interrupt.csv' + result_path_node = os.path.join(dest_dir,node_name,result_name) + result_path_conf = os.path.join(dest_dir.replace('raw','conf'),result_name) s_l = [] e_l = [] diff_list = [] @@ -548,25 +548,42 @@ def interrupt_diff(self,dest_dir,node_name,s_path,e_path): lines = [] for j in range(len(s_l[i])): if s_l[i][j].isdigit() and e_l[i][j].isdigit(): - diff_value = int(e_l[i][j]) - int(s_l[i][j]) lines.append(int(e_l[i][j]) - int(s_l[i][j])) else: lines.append(e_l[i][j]) diff_list.append(lines) - if os.path.exists(result_path): - os.remove(result_path) - output = open(result_path,'w+') - for line in diff_list: - line_str = '' - for col in range(len(line)): - if col != len(line)-1: - line_str += str(line[col])+' ' - else: - line_str += str(line[col]) - output.writelines(line_str) - output.close() + ##write interrupt to node and conf + common.printout("LOG","write interrput to node and conf.") + if os.path.exists(result_path_node): + os.remove(result_path_node) + if os.path.exists(result_path_conf): + os.remove(result_path_conf) + output_node = file(result_path_node,'wb') + output_conf = file(result_path_conf,'wb') + interrupt_csv_node = csv.writer(output_node) + interrupt_csv_conf = csv.writer(output_conf) + if len(diff_list) != 0: + diff_list[0][0] = "" + interrupt_csv_node.writerow(diff_list[0]) + interrupt_csv_conf.writerow(diff_list[0]) + del diff_list[0] + new_diff_list = self.delete_colon(diff_list) + for i in new_diff_list: + interrupt_csv_node.writerows([i]) + interrupt_csv_conf.writerows([i]) + output_node.close() + output_conf.close() + else: + common.printout("WARNING","no interrupt.") else: - print 'ERROR: interrupt_start lines and interrupt_end lines are diffrent ! can not calculate diffrent value!' + common.printout("ERROR",'interrupt_start lines and interrupt_end lines are different ! can not calculate different value!') + + def delete_colon(self,data_list): + self.d_list = data_list + for i in range(len(self.d_list)): + self.d_list[i][0] = self.d_list[i][0].replace(":","") + self.d_list[i][-1] = self.d_list[i][-1].strip("\n") + return self.d_list def check_interrupt(self,s_inter,e_inter): result = "True" diff --git a/analyzer/analyzer_remote.py b/analyzer/analyzer_remote.py index 9841b99..25613c9 100644 --- a/analyzer/analyzer_remote.py +++ b/analyzer/analyzer_remote.py @@ -15,6 +15,7 @@ import copy import config from multiprocessing import Process +import csv pp = pprint.PrettyPrinter(indent=4) class Analyzer: @@ -389,8 +390,8 @@ def _process_data(self): if '_interrupts_end.txt' in dir_name: self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) if os.path.exists("%s/%s" % (dest_dir, dir_name.replace('end','start'))): - interrupt_start = "%s/%s" % (dest_dir, dir_name) - interrupt_end = "%s/%s" % (dest_dir, dir_name.replace('end','start')) + interrupt_end = "%s/%s" % (dest_dir, dir_name) + interrupt_start = "%s/%s" % (dest_dir, dir_name.replace('end','start')) self.interrupt_diff(dest_dir,self.whoami,interrupt_start,interrupt_end) if '_process_log.txt' in dir_name: self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) @@ -420,8 +421,8 @@ def process_smartinfo_data(self, path): def interrupt_diff(self,dest_dir,node_name,s_path,e_path): s_p = s_path e_p = e_path - result_name = node_name+'_interrupt.txt' - result_path = os.path.join(dest_dir.replace('raw','conf'),result_name) + result_name = node_name+'_interrupt.csv' + result_path_node = os.path.join(dest_dir,result_name) s_l = [] e_l = [] diff_list = [] @@ -446,25 +447,35 @@ def interrupt_diff(self,dest_dir,node_name,s_path,e_path): lines = [] for j in range(len(s_l[i])): if s_l[i][j].isdigit() and e_l[i][j].isdigit(): - diff_value = int(e_l[i][j]) - int(s_l[i][j]) lines.append(int(e_l[i][j]) - int(s_l[i][j])) else: lines.append(e_l[i][j]) diff_list.append(lines) - if os.path.exists(result_path): - os.remove(result_path) - output = open(result_path,'w+') - for line in diff_list: - line_str = '' - for col in range(len(line)): - if col != len(line)-1: - line_str += str(line[col])+' ' - else: - line_str += str(line[col]) - output.writelines(line_str) - output.close() + ##write interrupt to node and conf + self.common.printout("LOG","write interrput to node and conf.") + if os.path.exists(result_path_node): + os.remove(result_path_node) + output_node = file(result_path_node,'wb') + interrupt_csv_node = csv.writer(output_node) + if len(diff_list) != 0: + diff_list[0][0] = "" + interrupt_csv_node.writerow(diff_list[0]) + del diff_list[0] + new_diff_list = self.delete_colon(diff_list) + for i in new_diff_list: + interrupt_csv_node.writerows([i]) + output_node.close() + else: + self.common.printout("WARNING","no interrupt.") else: - print 'ERROR: interrupt_start lines and interrupt_end lines are diffrent ! can not calculate diffrent value!' + self.common.printout("ERROR",'interrupt_start lines and interrupt_end lines are different ! can not calculate different value!') + + def delete_colon(self,data_list): + self.d_list = data_list + for i in range(len(self.d_list)): + self.d_list[i][0] = self.d_list[i][0].replace(":","") + self.d_list[i][-1] = self.d_list[i][-1].strip("\n") + return self.d_list def check_interrupt(self,s_inter,e_inter): result = "True" diff --git a/benchmarking/mod/benchmark.py b/benchmarking/mod/benchmark.py index ab845b3..5241629 100644 --- a/benchmarking/mod/benchmark.py +++ b/benchmarking/mod/benchmark.py @@ -252,6 +252,8 @@ def archive(self): for node in self.cluster["osd"]: common.bash("mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) if "blktrace" in self.cluster["collector"]: common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*blktrace*" % self.cluster["tmp_dir"]) if "lttng" in self.cluster["collector"]: @@ -261,6 +263,8 @@ def archive(self): for node in self.benchmark["distribution"].keys(): common.bash( "mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) #save real runtime if self.real_runtime: