From 85b391fb7c076ee3eb9425af6712577d7c9568f6 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Mar 2017 14:11:37 +0800 Subject: [PATCH 1/2] analyzer json fio log Signed-off-by: root --- analyzer/analyzer.py | 15 +++++++++++++++ analyzer/analyzer_remote.py | 2 ++ benchmarking/mod/bblock/fiorbd.py | 2 +- benchmarking/mod/benchmark.py | 2 ++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/analyzer/analyzer.py b/analyzer/analyzer.py index 6ae9161..63fd84b 100644 --- a/analyzer/analyzer.py +++ b/analyzer/analyzer.py @@ -486,6 +486,9 @@ def _process_data(self, node_name): self.workpool.schedule( self.process_sar_data, "%s/%s/%s" % (dest_dir, node_name, dir_name)) if 'totals.html' in dir_name: self.workpool.schedule( self.process_vdbench_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), "%s_%s" % (node_name, dir_name)) + if '_fio.json' in dir_name: + print '===============' + self.workpool.schedule( self.process_json_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) if '_fio.txt' in dir_name: self.workpool.schedule( self.process_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) if '_fio_iops.1.log' in dir_name or '_fio_bw.1.log' in dir_name or '_fio_lat.1.log' in dir_name: @@ -844,6 +847,15 @@ def get_lat_persent_dict(self,fio_str): lat_percent_dict[key[0]] = value[0] return lat_percent_dict + def process_json_fio_data(self,path,dirname): + try: + import pdb + pdb.set_trace() + fio_data = json.loads(path, object_pairs_hook=OrderedDict) + print fio_data + except: + pass + def process_fio_data(self, path, dirname): result = {} stdout, stderr = common.bash("grep \" *io=.*bw=.*iops=.*runt=.*\|^ *lat.*min=.*max=.*avg=.*stdev=.*\" "+path, True) @@ -916,6 +928,9 @@ def process_fio_data(self, path, dirname): result[dirname]["fio"] = output_fio_data self.workpool.enqueue_data( ["process_fio_data", result] ) + print '---------------------------------' + print path,dirname + print result return result def process_lttng_data(self, path): diff --git a/analyzer/analyzer_remote.py b/analyzer/analyzer_remote.py index 14d225f..71e5127 100644 --- a/analyzer/analyzer_remote.py +++ b/analyzer/analyzer_remote.py @@ -796,6 +796,8 @@ def process_fio_data(self, path, dirname): result[dirname] = {} result[dirname]["fio"] = output_fio_data self.workpool.enqueue_data(["process_fio_data", result]) + print '=======================================' + print result return result def process_lttng_data(self, path): diff --git a/benchmarking/mod/bblock/fiorbd.py b/benchmarking/mod/bblock/fiorbd.py index f8299a8..76131d2 100644 --- a/benchmarking/mod/bblock/fiorbd.py +++ b/benchmarking/mod/bblock/fiorbd.py @@ -93,7 +93,7 @@ def run(self): poolname = self.benchmark["poolname"] for client in self.benchmark["distribution"]: rbdlist = ' '.join(self.benchmark["distribution"][client]) - res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --output %s/`hostname`_${rbdname}_fio.txt --write_bw_log=%s/`hostname`_${rbdname}_fio --write_lat_log=%s/`hostname`_${rbdname}_fio --write_iops_log=%s/`hostname`_${rbdname}_fio --section %s %s/fio.conf 2>%s/`hostname`_${rbdname}_fio_errorlog.txt & done" % (rbdlist, poolname, dest_dir, dest_dir, dest_dir, dest_dir, self.benchmark["section_name"], dest_dir, dest_dir), option = "force") + res = common.pdsh(user, [client], "for rbdname in %s; do POOLNAME=%s RBDNAME=${rbdname} fio --output-format=json --output %s/`hostname`_${rbdname}_fio.json --write_bw_log=%s/`hostname`_${rbdname}_fio --write_lat_log=%s/`hostname`_${rbdname}_fio --write_iops_log=%s/`hostname`_${rbdname}_fio --section %s %s/fio.conf 2>%s/`hostname`_${rbdname}_fio_errorlog.json & done" % (rbdlist, poolname, dest_dir, dest_dir, dest_dir, dest_dir, self.benchmark["section_name"], dest_dir, dest_dir), option = "force") fio_job_num_total += len(self.benchmark["distribution"][client]) self.chkpoint_to_log("fio start") time.sleep(1) diff --git a/benchmarking/mod/benchmark.py b/benchmarking/mod/benchmark.py index 54ca676..ac544b0 100644 --- a/benchmarking/mod/benchmark.py +++ b/benchmarking/mod/benchmark.py @@ -259,6 +259,7 @@ def archive(self): common.bash("mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.json" % self.cluster["tmp_dir"]) common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) if "blktrace" in self.cluster["collector"]: common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*blktrace*" % self.cluster["tmp_dir"]) @@ -270,6 +271,7 @@ def archive(self): common.bash( "mkdir -p %s/raw/%s" % (dest_dir, node)) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.txt" % self.cluster["tmp_dir"]) common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.csv" % self.cluster["tmp_dir"]) + common.rscp(user, node, "%s/raw/%s/" % (dest_dir, node), "%s/*.json" % self.cluster["tmp_dir"]) common.rscp(user, node, "%s/conf/" % (dest_dir), "%s/*.csv" % self.cluster["tmp_dir"]) #save real runtime From 0ea315733d58e17caeab5cbc168efbfc39b03816 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 21 Mar 2017 09:59:16 +0800 Subject: [PATCH 2/2] analyzer json fio log Signed-off-by: root --- analyzer/analyzer.py | 46 +++++++++++++++++++++++++++++++------ analyzer/analyzer_remote.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/analyzer/analyzer.py b/analyzer/analyzer.py index 63fd84b..366502a 100644 --- a/analyzer/analyzer.py +++ b/analyzer/analyzer.py @@ -487,8 +487,8 @@ def _process_data(self, node_name): if 'totals.html' in dir_name: self.workpool.schedule( self.process_vdbench_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), "%s_%s" % (node_name, dir_name)) if '_fio.json' in dir_name: - print '===============' - self.workpool.schedule( self.process_json_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) + if '.tmp' not in dir_name: + self.workpool.schedule( self.process_json_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) if '_fio.txt' in dir_name: self.workpool.schedule( self.process_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) if '_fio_iops.1.log' in dir_name or '_fio_bw.1.log' in dir_name or '_fio_lat.1.log' in dir_name: @@ -848,13 +848,45 @@ def get_lat_persent_dict(self,fio_str): return lat_percent_dict def process_json_fio_data(self,path,dirname): + result = {} try: - import pdb - pdb.set_trace() - fio_data = json.loads(path, object_pairs_hook=OrderedDict) - print fio_data + js={} + f = open(path,'r') + a=f.readlines() + path_tmp = path+'.tmp' + dir_name = os.path.dirname(path) + os.system('rm %s/*.tmp'%dir_name) + f_tmp=open(path_tmp,'w') + b=''.join(a[1:]) + f_tmp.write(b) + f.close() + f_tmp.close() + with open(path_tmp,'r') as f_json: + js.update(json.load(f_json,object_pairs_hook=OrderedDict)) + output_fio_data = OrderedDict() + output_fio_data['read_lat'] = js['jobs'][0]["read"]['lat']['mean'] + output_fio_data['read_iops'] = js['jobs'][0]["read"]['iops'] + output_fio_data['read_bw'] = js['jobs'][0]["read"]['bw'] + output_fio_data['read_runtime'] = js['jobs'][0]["read"]['runtime'] + output_fio_data['write_lat'] = js['jobs'][0]["write"]['lat']['mean'] + output_fio_data['write_iops'] = js['jobs'][0]["write"]['iops'] + output_fio_data['write_bw'] = js['jobs'][0]["write"]['bw'] + output_fio_data['write_runtime'] = js['jobs'][0]["write"]['runtime'] + if js['jobs'][0]["read"]['clat']['percentile']['95.000000'] == 0: + output_fio_data['95.00th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['95.000000'] + if js['jobs'][0]["read"]['clat']['percentile']['99.000000'] == 0: + output_fio_data['99.00th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['99.000000'] + if js['jobs'][0]["read"]['clat']['percentile']['99.990000'] == 0: + output_fio_data['99.99th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['99.990000'] + output_fio_data['lat_unit'] = 'msec' + output_fio_data['runtime_unit'] = 'sec' + output_fio_data['bw_unit'] = 'MB/s' + result[dirname] = {} + result[dirname]["fio"] = output_fio_data + self.workpool.enqueue_data( ["process_json_fio_data", result] ) + return result except: - pass + return result def process_fio_data(self, path, dirname): result = {} diff --git a/analyzer/analyzer_remote.py b/analyzer/analyzer_remote.py index 71e5127..f0adcdf 100644 --- a/analyzer/analyzer_remote.py +++ b/analyzer/analyzer_remote.py @@ -367,6 +367,9 @@ def _process_data(self): if 'totals.html' in dir_name: self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) self.workpool.schedule( self.process_vdbench_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), "%s_%s" % (node_name, dir_name)) + if '_fio.json' in dir_name: + if '.tmp' not in dir_name: + self.workpool.schedule( self.process_json_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) if '_fio.txt' in dir_name: self.common.printout("LOG","Processing %s_%s" % (self.whoami, dir_name)) self.workpool.schedule( self.process_fio_data, "%s/%s/%s" % (dest_dir, node_name, dir_name), dir_name) @@ -725,6 +728,47 @@ def get_lat_persent_dict(self,fio_str): lat_percent_dict[key[0]] = value[0] return lat_percent_dict + def process_json_fio_data(self,path,dirname): + result = {} + try: + js={} + f = open(path,'r') + a=f.readlines() + path_tmp = path+'.tmp' + dir_name = os.path.dirname(path) + os.system('rm %s/*.tmp'%dir_name) + f_tmp=open(path_tmp,'w') + b=''.join(a[1:]) + f_tmp.write(b) + f.close() + f_tmp.close() + with open(path_tmp,'r') as f_json: + js.update(json.load(f_json,object_pairs_hook=OrderedDict)) + output_fio_data = OrderedDict() + output_fio_data['read_lat'] = js['jobs'][0]["read"]['lat']['mean'] + output_fio_data['read_iops'] = js['jobs'][0]["read"]['iops'] + output_fio_data['read_bw'] = js['jobs'][0]["read"]['bw'] + output_fio_data['read_runtime'] = js['jobs'][0]["read"]['runtime'] + output_fio_data['write_lat'] = js['jobs'][0]["write"]['lat']['mean'] + output_fio_data['write_iops'] = js['jobs'][0]["write"]['iops'] + output_fio_data['write_bw'] = js['jobs'][0]["write"]['bw'] + output_fio_data['write_runtime'] = js['jobs'][0]["write"]['runtime'] + if js['jobs'][0]["read"]['clat']['percentile']['95.000000'] == 0: + output_fio_data['95.00th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['95.000000'] + if js['jobs'][0]["read"]['clat']['percentile']['99.000000'] == 0: + output_fio_data['99.00th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['99.000000'] + if js['jobs'][0]["read"]['clat']['percentile']['99.990000'] == 0: + output_fio_data['99.99th%_lat'] = js['jobs'][0]["write"]['clat']['percentile']['99.990000'] + output_fio_data['lat_unit'] = 'msec' + output_fio_data['runtime_unit'] = 'sec' + output_fio_data['bw_unit'] = 'MB/s' + result[dirname] = {} + result[dirname]["fio"] = output_fio_data + self.workpool.enqueue_data( ["process_json_fio_data", result] ) + return result + except: + return result + def process_fio_data(self, path, dirname): result = {} stdout = self.common.bash("grep \" *io=.*bw=.*iops=.*runt=.*\|^ *lat.*min=.*max=.*avg=.*stdev=.*\" "+path)