diff --git a/core/waf/common.py b/core/waf/common.py index f3012aea..b44e79cc 100644 --- a/core/waf/common.py +++ b/core/waf/common.py @@ -7,7 +7,7 @@ from core.tools.repository import read_repos from waflib import Errors, Context, Utils, Options, Build, Configure, TaskGen -def options(self): +def options(self): """Setting default waf options right for SoCRocket""" #common = self.get_option_group("--prefix") #common.set_title("Common Configuration Options") @@ -21,9 +21,9 @@ def options(self): # """All following options can be provided to the configure rule.""" #) self.add_option( - "--verbosity", - dest="verbosity", - help="Defines the verbosity for the build", + "--verbosity", + dest="verbosity", + help="Defines the verbosity for the build", default=os.environ.get("VERBOSITY",'4') ) @@ -31,7 +31,7 @@ def configure(self): """Standard tools needed for fetching and building""" print(""" To compile SoCRocket you need to have installed a basic Unix system. - Atleast it needs to contain: + Atleast it needs to contain: nm, git, patch, make, wget / curl, tar, ln, bash ans swig, python. @@ -40,7 +40,7 @@ def configure(self): libreadline-dev Furthermore you need to provide cross compiler for sparc. If not provided - we will try to fetch them from Gaisler (sparc-elf-gcc). But you will need + we will try to fetch them from Gaisler (sparc-elf-gcc). But you will need to provice a 32bit environment to execute the precompiled toolchain. For the ipython support blas and lapack needs to be installed. @@ -55,6 +55,13 @@ def configure(self): self.find_program('tar', var='TAR', mandatory=True, okmsg="ok") self.find_program('ln', var='LN', mandatory=True, okmsg="ok") self.find_program('bash', var='BASH', mandatory=True, okmsg="ok") + self.find_program('perf', var='PERF', mandatory=False, okmsg="ok") + if 'PERF' not in self.env: + print (""" + For any performance record with perf you have to install: + + linux-tools-common linux-tools-generic linux-tools-`uname -r` + """) if self.options.jobs: self.env["JOBS"] = "-j%d" % self.options.jobs @@ -90,15 +97,15 @@ def fun(*k,**kw): # Find all project targets: # ./waf list # -#def target_list(ctx): +#def target_list(ctx): # """returns a list of all targets""" # -# bld = Build.BuildContext() -# proj = Environment.Environment(Options.lockfile) -# bld.load_dirs(proj['srcdir'], proj['blddir']) +# bld = Build.BuildContext() +# proj = Environment.Environment(Options.lockfile) +# bld.load_dirs(proj['srcdir'], proj['blddir']) # bld.load_envs() # -# bld.add_subdirs([os.path.split(Utils.g_module.root_path)[0]]) +# bld.add_subdirs([os.path.split(Utils.g_module.root_path)[0]]) # # nlibs = set([]) # ntests = set([]) @@ -111,14 +118,14 @@ def fun(*k,**kw): # napps.add(x.name or x.target) # elif "cstaticlib" in x.features: # nlibs.add(x.name or x.target) -# +# # except AttributeError: # pass # # libs = list(nlibs) # tests = list(ntests) # apps = list(napps) -# +# # libs.sort() # tests.sort() # apps.sort() @@ -131,7 +138,7 @@ def fun(*k,**kw): # for name in tests: # print " ", name # print "" -# +# # print "Executable targets:" # for name in apps: # print " ", name @@ -149,7 +156,7 @@ def get_subdirs(self,path='.'): """Return a list of all subdirectories from path""" REPOS = read_repos(Context.top_dir) result = [name - for name in os.listdir(path) + for name in os.listdir(path) if os.path.isfile(os.path.join(path, name, "wscript")) and (not os.path.relpath( os.path.join(path,name), Context.top_dir) in REPOS.keys() or name == 'core') and not name.startswith('.')] return result @@ -167,8 +174,8 @@ def recurse_all_tests(self): def get_testdirs(path='.'): """Return a list of all test subdirectories of path""" - return [os.path.join(name, "tests") - for name in os.listdir(path) + return [os.path.join(name, "tests") + for name in os.listdir(path) if os.path.isfile( os.path.join(path, name, "tests", "wscript"))] @@ -176,7 +183,7 @@ def getdirs(base = '.', excludes = list()): """ For external resources Not used at the moment, maybe with grlib - Return recursively all subdirectories of base, + Return recursively all subdirectories of base, exept directories matching excludes. """ result = [] @@ -193,13 +200,13 @@ def getfiles(base = '.', ext = list('*'), excludes = list()): for cdir in dirs: if any([fnmatch.fnmatchcase(os.path.join(root, cdir), exc) for exc in excludes]): del(cdir) - + for cfile in files: if any([fnmatch.fnmatchcase(os.path.join(root, cfile), exc) for exc in excludes]): continue - if any([fnmatch.fnmatchcase(os.path.join(root, cfile), e) for e in ext]): + if any([fnmatch.fnmatchcase(os.path.join(root, cfile), e) for e in ext]): result.append(os.path.join(root, cfile)) - + return result @TaskGen.before('process_source', 'process_rule') @@ -209,4 +216,3 @@ def export_has_define(self): defines = Utils.to_list(defines) defines += ["HAVE_" + self.target.replace(".", "_").upper()] setattr(self, "export_defines", defines) - diff --git a/core/waf/flamegraph.py b/core/waf/flamegraph.py new file mode 100644 index 00000000..7b054216 --- /dev/null +++ b/core/waf/flamegraph.py @@ -0,0 +1,66 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vim: set expandtab:ts=4:sw=4:setfiletype python +import os +import subprocess +from waflib.Errors import ConfigurationError + +if "check_output" not in dir( subprocess ): # duck punch it in! + def f(*popenargs, **kwargs): + if 'stdout' in kwargs: + raise ValueError('stdout argument not allowed, it will be overridden.') + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + raise subprocess.CalledProcessError(retcode, cmd) + return output + subprocess.check_output = f + +def options(self): + self.add_option( + '--with-flamegraph', + type='string', + help='Basedir of your FlameGraph installation', + dest='flamegraphdir', + default=os.environ.get("FLAMEGRAPH_HOME") + ) + +def find(self, path = None): + if path: + path = os.path.abspath(os.path.expanduser(os.path.expandvars(os.path.join(path, "bin")))) + + self.find_program('flamegraph.pl', var='FLAMEGRAPH', mandatory=True, okmsg="ok") + + if "FLAMEGRAPH" in self.env: + self.start_msg("Checking flamegraph version") + flamegraph_version_str = subprocess.check_output(["flamegraph", "--version"]) + flamegraph_version_str = flamegraph_version_str.decode('utf-8').split('\n')[0] + flamegraph_version = [int(v) for v in flamegraph_version_str.split(" ")[1].split(".")] + flamegraph_version = flamegraph_version[0] * 1000000 + flamegraph_version[1] * 10000 + + if not ("FLAMEGRAPH" in self.env) or flamegraph_version < 1000000: + self.end_msg(flamegraph_version_str[:-2] + " is not enough using internal version") + self.fatal("You need at least FLAMEGRAPH version 1.0") + else: + self.end_msg(flamegraph_version_str[:-2] + " is ok") + +def configure(self): + try: + if self.options.flamegraphdir: + find(self, self.options.flamegraphdir) + else: + find(self) + except ConfigurationError as e: + name = "FlameGraph" + version = "1.0" + self.dep_fetch( + name = name, + version = version, + tar_url = "https://github.com/brendangregg/FlameGraph/archive/v1.0.tar.gz", + ) + self.find_program('flamegraph.pl', var='FLAMEGRAPH', mandatory=True, okmsg="ok", + path_list=[os.path.join(self.dep_path(name, version))]) diff --git a/core/wscript b/core/wscript index 9da65d2c..d79f5bcb 100644 --- a/core/wscript +++ b/core/wscript @@ -31,7 +31,8 @@ REPOSITORY_TOOLS = [ 'cpplint', 'oclint', 'clang_compilation_database', - 'sparcelf' + 'sparcelf', + 'flamegraph' ] def build(self): diff --git a/performance_test.py b/performance_test.py new file mode 100644 index 00000000..2e176d24 --- /dev/null +++ b/performance_test.py @@ -0,0 +1,155 @@ +#!/usr/bin/python +import sys +import os +import subprocess +import argparse + +#Script for performance testing for SoCRocket +#TODO automatisches downloaden und integrieren vom flamegraph repository +#TODO dass perf auch installiert ist +# vgl. core/waf/cmake.py +#TODO gephi tool oder xml vorlage --> not possible +#TODO perf.data sodass user damit arbeiten kann, ohne sudo +#TODO via python. "chrown -R {}" + +def flame( name, rootuser): + with open(name + ".perf", "w") as flame_perf_file,open(name + ".perf.folded", "w") as flame_perf_folded_file,open(name + "_flamegraph.svg", "w") as flame_image_file: + print "generate perf report with c++filt" + if rootuser: + fg_p1 = subprocess.Popen([#"sudo", #needed to write perf.data + "perf", + "script"], #saves perf results in perf.data + stdout=subprocess.PIPE) + else: + fg_p1 = subprocess.Popen(["sudo", #needed to write perf.data + "perf", + "script"], #saves perf results in perf.data + stdout=subprocess.PIPE) + + fg_p2 = subprocess.Popen(["c++filt"], + stdin=fg_p1.stdout, + stdout=flame_perf_file) + fg_p2.communicate()[0] + fg_p2.wait() + if (0==os.stat(flame_perf_file.name).st_size): + fg_p1 = subprocess.Popen(["sudo", #needed to write perf.data + "perf", + "script"], #saves perf results in perf.data + stdout=subprocess.PIPE) + fg_p2 = subprocess.Popen(["c++filt"], + stdin=fg_p1.stdout, + stdout=flame_perf_file) + print "fold report data" + print wd+flame_perf_file.name + fg_p3 = subprocess.Popen([wd+"build/.conf_check_deps/src/FlameGraph-1.0/stackcollapse-perf.pl", wd+flame_perf_file.name], stdout=flame_perf_folded_file) + fg_p3.wait() + print "generate image" + fg_p4 = subprocess.Popen([wd+"build/.conf_check_deps/src/FlameGraph-1.0/flamegraph.pl", wd+flame_perf_folded_file.name], stdout=flame_image_file) + fg_p4.wait() + +def dot( name, rootuser): + #perf script | c++filt | gprof2dot.py -f perf | dot -Tpng -o output.png + cg_image_file= name+"_callgraph.png" + if rootuser: + cg_p1 = subprocess.Popen([#"sudo", + "perf", + "script" #saves perf results in perf.data + ], stdout=subprocess.PIPE) + else: #TODO or cg_image_file.size=0 + cg_p1 = subprocess.Popen(["sudo", + "perf", + "script" #saves perf results in perf.data + ], stdout=subprocess.PIPE) + cg_p2 = subprocess.Popen(["c++filt"], + stdin=cg_p1.stdout, + stdout=subprocess.PIPE) + if (0==os.stat(cg_image_file).st_size): + cg_p1 = subprocess.Popen(["sudo", + "perf", + "script" #saves perf results in perf.data + ], stdout=subprocess.PIPE) + cg_p2 = subprocess.Popen(["c++filt"], + stdin=cg_p1.stdout, + stdout=subprocess.PIPE) + print "generating dot graph of "+name + cg_p3 = subprocess.Popen([ #subprocess.check_output() + "gprof2dot", # creates a dot graph + "-f", "perf", #format perf + "--strip", #strip funktction parameters + "--wrap" #wrap function names + ], stdin=cg_p2.stdout, stdout=subprocess.PIPE) + cg_p4 = subprocess.check_output(("dot -Tpng -o "+ cg_image_file).split(), stdin=cg_p3.stdout) + +def txt(name,rootuser): + #perf report --sort comm,dso | c++filt >> output.txt + print "Generating txt" + with open(name+"_report.txt","w") as cg_report_file: + if rootuser: + txt_p1 = subprocess.Popen([#"sudo", + "perf", + "report", + "--sort", + "comm,dso"], stdout=subprocess.PIPE) + else: + txt_p1 = subprocess.Popen(["sudo", + "perf", + "report", + "--sort", + "comm,dso"], stdout=subprocess.PIPE) + txt_p2 = subprocess.Popen("c++filt", stdin=txt_p1.stdout, stdout=cg_report_file) + txt_p2.wait(); + if (0==os.stat(cg_report_file.name).st_size): + txt_p1 = subprocess.Popen(["sudo", + "perf", + "report", + "--sort", + "comm,dso"], stdout=subprocess.PIPE) + txt_p2 = subprocess.Popen("c++filt", stdin=txt_p1.stdout, stdout=cg_report_file) + txt_p2.wait(); + + +print "SocRocket Performance" + +parser = argparse.ArgumentParser() +parser.add_argument('filename') +parser.add_argument("--asroot", help="run this script as root", action="store_true") +parser.add_argument("--asuser", help="run this script as user", action="store_true") +args = parser.parse_args() + +#get path from first argument +origin = args.filename +while (None == origin): + origin = raw_input("Path? ") +origin_norm= os.path.realpath(origin) +#get filename from path +name = os.path.basename(origin_norm) +#update current working directory +wd=os.getcwd()+os.sep +# +asroot = False +if args.asroot: + asroot = True +print "collecting performance data of %s" %(origin_norm) +if asroot: + perf_record = "perf record --call-graph dwarf -F 99 %s" %(origin_norm) +else: + perf_record = "sudo perf record --call-graph dwarf -F 99 %s" %(origin_norm) +#kernel.perf_event_paranoid +print perf_record +fg_p0 = subprocess.Popen(perf_record.split()) +fg_p0.wait() +#repeat +#get user permission +cwd=os.getcwd() +print cwd +if os.stat('perf.data').st_size>0: + #post_processing + flame(name,asroot) + dot(name,asroot) + txt(name,asroot) + print "deleting system files" + os.remove("perf.data") + os.remove(name+".perf") + os.remove(name+".perf.folded") +else: + print "No samples recorded" diff --git a/pysc/waf/bpython.py b/pysc/waf/bpython.py new file mode 100644 index 00000000..35acd80a --- /dev/null +++ b/pysc/waf/bpython.py @@ -0,0 +1,72 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vim: set expandtab:ts=4:sw=4:setfiletype python +import os +import stat +from waflib import Context +from waflib.Build import BuildContext +import atexit + +def options(self): + pass + +packages = [ + "bpython" +] + +# packages for perf +# sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` + +# sudo apt install python-pip +# pip install gprof2dot + +ACTIVATE = """ +#!/bin/bash +. ~/.bashrc +. %(venv)s/bin/activate +export LD_LIBRARY_PATH="%(libpath)s:$LD_LIBRARY_PATH" +export PYTHONPATH="%(pythonpath)s:$PYTHONPATH" +export PATH="%(path)s:$PATH" +alias deactivate=exit +""" + +def configure(self): + + for package in packages: + self.python_get(package) + +def vexecute(self, cmd = ""): + path = [] + libpath = [] + pythonpath = [] + for key in list(self.env.table.keys()): + if key.startswith("PATH_"): + for val in self.env[key]: + if not val in path: + path.append(val) + if key.startswith("LIBPATH_") and not key == "LIBPATH_ST": + for val in self.env[key]: + if not val in libpath: + libpath.append(val) + if key.startswith("PYTHONPATH_"): + for val in self.env[key]: + if not val in pythonpath: + pythonpath.append(val) + path.append(os.path.join(self.srcnode.abspath(), "tools")) + pythonpath.append(self.srcnode.abspath()) + + activate = self.bldnode.find_or_declare("activate") + activate.write(ACTIVATE % {"path" : ':'.join(path), "libpath" : ':'.join(libpath), "pythonpath" : ':'.join(pythonpath), "venv": self.env.VENV_PATH}) + activate.chmod(stat.S_IXUSR | stat.S_IXGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IWUSR | stat.S_IWGRP) + def eexit(): + os.execve(self.env.BASH, [self.env.BASH , '--rcfile', activate.abspath(), '-i'], os.environ) + atexit.register(eexit) + +def bash(self): + vexecute(self) + +class Bash(BuildContext): + cmd = 'bash' + fun = 'bash' + +setattr(Context.g_module, 'bash', bash) diff --git a/pysc/waf/gprof2dot.py b/pysc/waf/gprof2dot.py new file mode 100644 index 00000000..74d91afa --- /dev/null +++ b/pysc/waf/gprof2dot.py @@ -0,0 +1,72 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# vim: set expandtab:ts=4:sw=4:setfiletype python +import os +import stat +from waflib import Context +from waflib.Build import BuildContext +import atexit + +def options(self): + pass + +packages = [ + "gprof2dot" +] + +# packages for perf +# sudo apt-get install linux-tools-common linux-tools-generic linux-tools-`uname -r` + +# sudo apt install python-pip +# pip install gprof2dot + +ACTIVATE = """ +#!/bin/bash +. ~/.bashrc +. %(venv)s/bin/activate +export LD_LIBRARY_PATH="%(libpath)s:$LD_LIBRARY_PATH" +export PYTHONPATH="%(pythonpath)s:$PYTHONPATH" +export PATH="%(path)s:$PATH" +alias deactivate=exit +""" + +def configure(self): + + for package in packages: + self.python_get(package) + +def vexecute(self, cmd = ""): + path = [] + libpath = [] + pythonpath = [] + for key in list(self.env.table.keys()): + if key.startswith("PATH_"): + for val in self.env[key]: + if not val in path: + path.append(val) + if key.startswith("LIBPATH_") and not key == "LIBPATH_ST": + for val in self.env[key]: + if not val in libpath: + libpath.append(val) + if key.startswith("PYTHONPATH_"): + for val in self.env[key]: + if not val in pythonpath: + pythonpath.append(val) + path.append(os.path.join(self.srcnode.abspath(), "tools")) + pythonpath.append(self.srcnode.abspath()) + + activate = self.bldnode.find_or_declare("activate") + activate.write(ACTIVATE % {"path" : ':'.join(path), "libpath" : ':'.join(libpath), "pythonpath" : ':'.join(pythonpath), "venv": self.env.VENV_PATH}) + activate.chmod(stat.S_IXUSR | stat.S_IXGRP | stat.S_IRUSR | stat.S_IRGRP | stat.S_IWUSR | stat.S_IWGRP) + def eexit(): + os.execve(self.env.BASH, [self.env.BASH , '--rcfile', activate.abspath(), '-i'], os.environ) + atexit.register(eexit) + +def bash(self): + vexecute(self) + +class Bash(BuildContext): + cmd = 'bash' + fun = 'bash' + +setattr(Context.g_module, 'bash', bash) diff --git a/pysc/wscript b/pysc/wscript index de8de112..518f91f9 100644 --- a/pysc/wscript +++ b/pysc/wscript @@ -10,6 +10,8 @@ REPOSITORY_TOOLS = [ "virtualenv", # "consolelog", "shell", + "bpython", + "gprof2dot" ] def build(self):