diff --git a/avaframe/ana5Utils/distanceTimeAnalysis.py b/avaframe/ana5Utils/distanceTimeAnalysis.py index 638c227b1..c5afe79eb 100644 --- a/avaframe/ana5Utils/distanceTimeAnalysis.py +++ b/avaframe/ana5Utils/distanceTimeAnalysis.py @@ -559,7 +559,7 @@ def extractFrontAndMeanValuesTT(cfgRangeTime, flowF, demHeader, mtiInfo): return mtiInfo -def initializeRangeTime(modName, cfg, dem, simHash): +def initializeRangeTime(modName, cfg, dem, simHash, configDir): """ initialize generation of range-time diagram for visualizing simulation data Parameters @@ -572,6 +572,8 @@ def initializeRangeTime(modName, cfg, dem, simHash): dictionary with DEM header and data simHash: str unique simulation ID + configDir: str or pathlib path + path to configuration directory - optional if not provided has to be empty string Returns -------- @@ -583,7 +585,7 @@ def initializeRangeTime(modName, cfg, dem, simHash): """ # fetch configuration and add info - cfgRangeTime = cfgUtils.getModuleConfig(modName) + cfgRangeTime = cfgUtils.getModuleConfig(modName, configDir) cfgRangeTime['GENERAL']['tEnd'] = cfg['GENERAL']['tEnd'] cfgRangeTime['GENERAL']['avalancheDir'] = cfg['GENERAL']['avalancheDir'] diff --git a/avaframe/avaframeCfg.ini b/avaframe/avaframeCfg.ini index 1e43f3fc3..5689d5526 100644 --- a/avaframe/avaframeCfg.ini +++ b/avaframe/avaframeCfg.ini @@ -5,7 +5,8 @@ [MAIN] # Path to avalanche directory avalancheDir = data/avaParabola - +# OPTIONAL Path to configuration file directory +configurationDir = # number of CPU cores to use for the computation of com1DFA # possible values are: # - auto -> takes up to CPUPercent (see below) % of the available CPU cores diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index f2e8d58c6..5ca3e636a 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -93,7 +93,7 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo): # read initial configuration if typeCfgInfo in ["cfgFromFile", "cfgFromDefault"]: - cfgStart = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgInfo, toPrint=False) + cfgStart = cfgUtils.getModuleConfig(com1DFA, cfgMain['MAIN']['configurationDir'], fileOverride=cfgInfo, toPrint=False) elif typeCfgInfo == "cfgFromObject": cfgStart = cfgInfo @@ -167,7 +167,7 @@ def com1DFAMain(cfgMain, cfgInfo=""): nCPU = cfgUtils.getNumberOfProcesses(cfgMain, len(simDict)) # Supply compute task with inputs - com1DFACoreTaskWithInput = partial(com1DFACoreTask, simDict, inputSimFiles, avalancheDir, outDir) + com1DFACoreTaskWithInput = partial(com1DFACoreTask, simDict, inputSimFiles, cfgMain, outDir) # Create parallel pool and run # with multiprocessing.Pool(processes=nCPU) as pool: @@ -200,11 +200,13 @@ def com1DFAMain(cfgMain, cfgInfo=""): return 0, {}, [], "" -def com1DFACoreTask(simDict, inputSimFiles, avalancheDir, outDir, cuSim): +def com1DFACoreTask(simDict, inputSimFiles, cfgMain, outDir, cuSim): """This is a subdivision of com1DFAMain to allow for parallel execution. Please read this in the context of the com1DFAMain function. """ + avalancheDir = cfgMain['MAIN']['avalancheDir'] + simDF = pd.DataFrame() tCPUDF = pd.DataFrame() @@ -232,7 +234,7 @@ def com1DFACoreTask(simDict, inputSimFiles, avalancheDir, outDir, cuSim): cfgFinal, tCPU, particlesList, - ) = com1DFA.com1DFACore(cfg, avalancheDir, cuSim, inputSimFiles, outDir, simHash=simHash) + ) = com1DFA.com1DFACore(cfg, cfgMain, cuSim, inputSimFiles, outDir, simHash=simHash) simDF.at[simHash, "nPart"] = str(int(particlesList[0]["nPart"])) @@ -323,7 +325,7 @@ def com1DFAPostprocess(simDF, tCPUDF, simDFExisting, cfgMain, dem, reportDictLis return dem, plotDict, reportDictList, simDFNew -def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): +def com1DFACore(cfg, cfgMain, cuSimName, inputSimFiles, outDir, simHash=""): """Run main com1DFA model This will compute a dense flow avalanche with the settings specified in cfg and the name cuSimName @@ -332,12 +334,12 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): ---------- cfg : configparser object configuration object for simulation to be performed + cfgMain: configparser object + main configuration of AvaFrame used here: avalancheDir, configurationDir cuSimName: str name of simulation inputSimFiles: dict dictionary with input files, release scenario chosen according to inputSimFiles['releaseScenario'] - avaDir : str or pathlib object - path to avalanche directory outDir: str or pathlib object path to Outputs simHash: str @@ -359,6 +361,8 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): list of particle dictionaries for all saving time steps """ + avaDir = cfgMain['MAIN']['avalancheDir'] + # select release area input data according to chosen release scenario inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg["INPUT"]["releaseScenario"]) @@ -388,7 +392,7 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): # ------------------------ # Start time step computation Tsave, particlesList, fieldsList, infoDict = DFAIterate( - cfg, particles, fields, dem, inputSimLines, simHash=simHash + cfg, particles, fields, dem, inputSimLines, cfgMain['MAIN']['configurationDir'], simHash=simHash ) # write mass balance to File @@ -1657,7 +1661,7 @@ def initializeResistance(cfg, dem, simTypeActual, resLine, reportAreaInfo, thres return cResRaster, detRaster, reportAreaInfo -def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): +def DFAIterate(cfg, particles, fields, dem, inputSimLines, configDir, simHash=""): """Perform time loop for DFA simulation Save results at desired intervals @@ -1676,6 +1680,8 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): dictionary with dem information inputSimLines : dict dictionary with input data dictionaries (releaseLine, entLine, ...) + configDir: str or pathlib Path + path to configuration directory - optional if not provided has to be empty string Returns ------- @@ -1762,7 +1768,7 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): # check if range-time diagram should be performed, if yes - initialize if cfg["VISUALISATION"].getboolean("createRangeTimeDiagram"): demRT = dtAna.setDemOrigin(dem) - mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash) + mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash, configDir) # fetch initial time step too mtiInfo, dtRangeTime = dtAna.fetchRangeTimeInfo( cfgRangeTime, cfg, dtRangeTime, t, demRT["header"], fields, mtiInfo diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index a80d41296..c27aa91b0 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -283,7 +283,8 @@ def createSimDictFromCfgs(cfgMain, cfgPath): # loop over all cfgFiles and create simDict for index, cfgFile in enumerate(cfgFilesAll): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile, toPrint=False) + # configDir is set to '' because fileOverride is provided + cfgFromFile = cfgUtils.getModuleConfig(com1DFA, '', fileOverride=cfgFile, toPrint=False) # create dictionary with one key for each simulation that shall be performed # NOTE: sims that are added don't need to be added to the simNameExisting list as diff --git a/avaframe/in3Utils/cfgHandling.py b/avaframe/in3Utils/cfgHandling.py index 9446ddb76..6f82e1d8e 100644 --- a/avaframe/in3Utils/cfgHandling.py +++ b/avaframe/in3Utils/cfgHandling.py @@ -540,6 +540,11 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): optional - path to directory to store local_ cfg ini file to if not provided - local_ cfg ini file is saved to avalanche directory + Returns + -------- + locFilePath: pathlib Path + path to directory where local configuration files are written to derived from override sections + """ # if a path is provided - save local cfg ini file there @@ -576,6 +581,7 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): cfgModule = cfgUtils.getModuleConfig( cfgNamePath, + '', fileOverride="", modInfo=False, toPrint=False, @@ -606,6 +612,8 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): log.info("%s CONFIGURATION wrote to %s" % (cfgName, str(cfgF))) + return locFilePath + def _removeCfgItemsNotInOverride(cfgModule, overrideKeys): """ remove options of cfgModule if not part of overrideKeys diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index 5a98b5e52..db11a4b58 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -1,7 +1,7 @@ -''' +""" Utilities for handling configuration files -''' +""" import configparser import logging @@ -27,25 +27,25 @@ log = logging.getLogger(__name__) -def getGeneralConfig(nameFile=''): - ''' Returns the general configuration for avaframe +def getGeneralConfig(nameFile=""): + """Returns the general configuration for avaframe returns a configParser object Parameters ---------- nameFile: pathlib path optional full path to file, if empty use avaframeCfg from folder one level up - ''' + """ # get path of module modPath = pathlib.Path(avaf.__file__).resolve().parent if isinstance(nameFile, pathlib.Path): - localFile = nameFile.parents[0] / ('local_' + nameFile.name) + localFile = nameFile.parents[0] / ("local_" + nameFile.name) defaultFile = nameFile else: - localFile = modPath / 'local_avaframeCfg.ini' - defaultFile = modPath / 'avaframeCfg.ini' + localFile = modPath / "local_avaframeCfg.ini" + defaultFile = modPath / "avaframeCfg.ini" if localFile.is_file(): iniFile = localFile @@ -55,16 +55,16 @@ def getGeneralConfig(nameFile=''): iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it - cfg, _ = readCompareConfig(iniFile, 'General', compare) + cfg, _ = readCompareConfig(iniFile, "General", compare) return cfg -def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDefault=False): - ''' Returns the configuration for a given module +def getModuleConfig(module, configDir, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): + """Returns the configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -76,6 +76,9 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe leads to getModuleConfig(c2) OR: pathlib Path to module (python file) + configDir: pathlib Path or str + directory to local configuration files - if not provided has to be empty str! + Str: fileOverride : allows for a completely different file location. However note: missing values from the default cfg will always be added! @@ -87,10 +90,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe Order is as follows: fileOverride -> local_MODULECfg.ini -> MODULECfg.ini - ''' + """ if isinstance(onlyDefault, bool) == False: - message = 'OnlyDefault parameter is not a boolean but %s' % type(onlyDefault) + message = "OnlyDefault parameter is not a boolean but %s" % type(onlyDefault) log.error(message) raise TypeError(message) @@ -101,11 +104,22 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe else: modPath, modName = getModPathName(module) - localFile = modPath / ('local_'+modName+'Cfg.ini') - defaultFile = modPath / (modName+'Cfg.ini') + if configDir != "": + if pathlib.Path(configDir).is_dir() is False: + message = "Provided configurationDir: %s is not a directory!" % str(configDir) + log.error(message) + raise NotADirectoryError(message) + else: + directoryFile = pathlib.Path(configDir, (("local_" + modName + "Cfg.ini"))) + else: + directoryFile = None - log.debug('localFile: %s', localFile) - log.debug('defaultFile: %s', defaultFile) + localFile = modPath / ("local_" + modName + "Cfg.ini") + defaultFile = modPath / (modName + "Cfg.ini") + + log.debug("localFile: %s", localFile) + log.debug("defaultFile: %s", defaultFile) + log.debug("directoryFile: %s", directoryFile) # Decide which one to take if fileOverride: @@ -114,9 +128,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = [defaultFile, fileOverride] compare = True else: - raise FileNotFoundError('Provided fileOverride does not exist: ' + - str(fileOverride)) - + raise FileNotFoundError("Provided fileOverride does not exist: " + str(fileOverride)) + elif (directoryFile is not None) and directoryFile.is_file(): + iniFile = [defaultFile, directoryFile] + compare = True elif localFile.is_file() and not onlyDefault: iniFile = localFile iniFile = [defaultFile, localFile] @@ -125,7 +140,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it cfg, modDict = readCompareConfig(iniFile, modName, compare, toPrint) @@ -137,7 +152,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe def getDefaultModuleConfig(module, toPrint=True): - ''' Returns the default configuration for a given module + """Returns the default configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -148,15 +163,14 @@ def getDefaultModuleConfig(module, toPrint=True): from avaframe.com2AB import com2AB as c2 leads to getModuleConfig(c2) - ''' + """ # get path to the module and its name modPath, modName = getModPathName(module) + defaultFile = modPath / (modName + "Cfg.ini") - defaultFile = modPath / (modName+'Cfg.ini') - - log.info('Getting the default config for %s', modName) - log.debug('defaultFile: %s', defaultFile) + log.info("Getting the default config for %s", modName) + log.debug("defaultFile: %s", defaultFile) # Finally read it cfg, _ = readCompareConfig(defaultFile, modName, compare=False, toPrint=toPrint) @@ -165,7 +179,7 @@ def getDefaultModuleConfig(module, toPrint=True): def readCompareConfig(iniFile, modName, compare, toPrint=True): - ''' Read and optionally compare configuration files (if a local and default are both provided) + """Read and optionally compare configuration files (if a local and default are both provided) and inform user of the eventual differences. Take the default as reference. Parameters @@ -184,10 +198,10 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): contains combined config modDict: dict dictionary containing only differences from default - ''' + """ if compare: - log.info('Reading config from: %s and %s' % (iniFile[0], iniFile[1])) + log.info("Reading config from: %s and %s" % (iniFile[0], iniFile[1])) # initialize configparser object to read defCfg = configparser.ConfigParser() defCfg.optionxform = str @@ -197,12 +211,12 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): defCfg.read(iniFile[0]) locCfg.read(iniFile[1]) - log.debug('Writing cfg for: %s', modName) + log.debug("Writing cfg for: %s", modName) # compare to default config and get modification dictionary and config modDict, modCfg = compareTwoConfigs(defCfg, locCfg, toPrint=toPrint) else: - log.info('Reading config from: %s', iniFile) + log.info("Reading config from: %s", iniFile) # initialize our final configparser object modCfg = configparser.ConfigParser() modCfg.optionxform = str @@ -217,58 +231,58 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): def _splitDeepDiffValuesChangedItem(inKey, inVal): - """ splits one item of a deepdiff result into section, key, old value, new value - - Parameters - ----------- - inputKey: str - key of a deepdiff changed_values item - inputValue: dict - value of a deepdiff changed_values item - - Returns - -------- - section: str - section name of changed item - key: str - key name of changed item - oldVal: str - old value - newVal: str - new value + """splits one item of a deepdiff result into section, key, old value, new value + + Parameters + ----------- + inputKey: str + key of a deepdiff changed_values item + inputValue: dict + value of a deepdiff changed_values item + + Returns + -------- + section: str + section name of changed item + key: str + key name of changed item + oldVal: str + old value + newVal: str + new value """ splitKey = re.findall(r"\['?([A-Za-z0-9_]+)'?\]", inKey) section = splitKey[0] key = splitKey[1] - return section, key, inVal['old_value'], inVal['new_value'] + return section, key, inVal["old_value"], inVal["new_value"] def compareTwoConfigs(defCfg, locCfg, toPrint=False): - """ compare locCfg to defCfg and return a cfg object and modification dict - Values are merged from locCfg to defCfg: - - parameters already in defCfg get the value from locCfg - - additional values in locCfg get added in the resulting Cfg - - Parameters - ----------- - defCfg: configparser object - default configuration - locCfg: configuration object - configuration that is compared to defCfg - toPrint: bool - flag if config shall be printed to log - - Returns - -------- - modInfo: dict - dictionary containing only differences from default - cfg: configParser object - contains combined config + """compare locCfg to defCfg and return a cfg object and modification dict + Values are merged from locCfg to defCfg: + - parameters already in defCfg get the value from locCfg + - additional values in locCfg get added in the resulting Cfg + + Parameters + ----------- + defCfg: configparser object + default configuration + locCfg: configuration object + configuration that is compared to defCfg + toPrint: bool + flag if config shall be printed to log + + Returns + -------- + modInfo: dict + dictionary containing only differences from default + cfg: configParser object + contains combined config """ - log.info('Comparing two configs') + log.info("Comparing two configs") # initialize modInfo and printOutInfo modInfo = dict() @@ -301,12 +315,12 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): # If toPrint is set, print full configuration: if toPrint: - for line in pformat(modCfgD, sort_dicts=False).split('\n'): + for line in pformat(modCfgD, sort_dicts=False).split("\n"): log.info(line) # Generate modInfo dictionary for output - if 'values_changed' in cfgDiff: - for key, value in cfgDiff['values_changed'].items(): + if "values_changed" in cfgDiff: + for key, value in cfgDiff["values_changed"].items(): section, itemKey, defValue, locValue = _splitDeepDiffValuesChangedItem(key, value) if section not in modInfo: @@ -316,91 +330,91 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): modInfo[section][itemKey] = modString # Log changes - log.info('COMPARING TO DEFAULT, THESE CHANGES HAPPENED:') - for line in cfgDiff.pretty().split('\n'): - log.info(line.replace('root','')) + log.info("COMPARING TO DEFAULT, THESE CHANGES HAPPENED:") + for line in cfgDiff.pretty().split("\n"): + log.info(line.replace("root", "")) return modInfo, modCfg -def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''): - """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini - or optional to filePath and with fileName +def writeCfgFile(avaDir, module, cfg, fileName="", filePath=""): + """Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini + or optional to filePath and with fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - cfg: configparser object - configuration settings - fileName: str - name of saved configuration file - optional - filePath: str or pathlib path - path where file should be saved to except file name - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + cfg: configparser object + configuration settings + fileName: str + name of saved configuration file - optional + filePath: str or pathlib path + path where file should be saved to except file name - optional """ # get filename of module name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set outputs - if filePath == '': - outDir = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles') + if filePath == "": + outDir = pathlib.Path(avaDir, "Outputs", modName, "configurationFiles") fU.makeADir(outDir) else: if filePath.is_dir(): outDir = pathlib.Path(filePath) else: - message = '%s is not a valid location for saving cfg file' % str(filePath) + message = "%s is not a valid location for saving cfg file" % str(filePath) log.error(message) raise NotADirectoryError(message) # set path to file - if fileName == '': + if fileName == "": fileName = modName - pathToFile = pathlib.Path(outDir, '%s.ini' % (fileName)) + pathToFile = pathlib.Path(outDir, "%s.ini" % (fileName)) # write file - with open(pathToFile, 'w') as conf: + with open(pathToFile, "w") as conf: cfg.write(conf) return pathToFile -def readCfgFile(avaDir, module='', fileName=''): - """ Read configuration from ini file, if module is provided, module configuration is read from Ouputs, - if fileName is provided configuration is read from fileName +def readCfgFile(avaDir, module="", fileName=""): + """Read configuration from ini file, if module is provided, module configuration is read from Ouputs, + if fileName is provided configuration is read from fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - fileName: str - path to file that should be read - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + fileName: str + path to file that should be read - optional - Returns - -------- - cfg: configParser object - configuration that is from file + Returns + -------- + cfg: configParser object + configuration that is from file """ # define file that should be read - if fileName != '': + if fileName != "": inFile = fileName - elif module != '': + elif module != "": # get module name name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set input file - inFile = pathlib.Path(avaDir, 'Outputs', '%s_settings.ini' % (modName)) + inFile = pathlib.Path(avaDir, "Outputs", "%s_settings.ini" % (modName)) else: - log.error('Please provide either a module or a fileName to read configuration from file') + log.error("Please provide either a module or a fileName to read configuration from file") raise NameError # read configParser object from input file, case sensitive @@ -413,7 +427,7 @@ def readCfgFile(avaDir, module='', fileName=''): def cfgHash(cfg, typeDict=False): - """ UID hash of a config. Given a configParser object cfg, + """UID hash of a config. Given a configParser object cfg, or a dictionary - then typeDict=True, returns a uid hash Parameters @@ -445,7 +459,7 @@ def cfgHash(cfg, typeDict=False): def convertConfigParserToDict(cfg): - """ create dictionary from configparser object """ + """create dictionary from configparser object""" cfgDict = {} for section in cfg.sections(): @@ -457,7 +471,7 @@ def convertConfigParserToDict(cfg): def convertDictToConfigParser(cfgDict): - """ create configParser object from dict """ + """create configParser object from dict""" cfg = configparser.ConfigParser() cfg.optionxform = str @@ -468,7 +482,7 @@ def convertDictToConfigParser(cfgDict): def writeDictToJson(inDict, outFilePath): - """ write a dictionary to a json file """ + """write a dictionary to a json file""" jsonDict = json.dumps(inDict, sort_keys=True, ensure_ascii=True) f = open(outFilePath, "w") @@ -476,54 +490,54 @@ def writeDictToJson(inDict, outFilePath): f.close() -def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir=''): - """ Read configurations from all simulations configuration ini files from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - standardCfg: dict - standard configuration for module - option - writeCSV: bool - True if configuration dataFrame shall be written to csv file - specDir: str - path to a directory where simulation configuration files can be found - optional - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations +def createConfigurationInfo(avaDir, comModule="com1DFA", standardCfg="", writeCSV=False, specDir=""): + """Read configurations from all simulations configuration ini files from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + standardCfg: dict + standard configuration for module - option + writeCSV: bool + True if configuration dataFrame shall be written to csv file + specDir: str + path to a directory where simulation configuration files can be found - optional + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', comModule, 'configurationFiles') - configFiles = inDir.glob('*.ini') + inDir = pathlib.Path(avaDir, "Outputs", comModule, "configurationFiles") + configFiles = inDir.glob("*.ini") if not inDir.is_dir(): - message = 'configuration file directory not found: %s' % (inDir) + message = "configuration file directory not found: %s" % (inDir) log.error(message) raise NotADirectoryError(message) elif configFiles == []: - message = 'No configuration file found in: %s' % (inDir) + message = "No configuration file found in: %s" % (inDir) log.error(message) raise FileNotFoundError(message) # create confiparser object, convert to json object, write to dataFrame # append all dataFrames - simDF = '' + simDF = "" for cFile in configFiles: - if 'sourceConfiguration' not in str(cFile): + if "sourceConfiguration" not in str(cFile): simName = pathlib.Path(cFile).stem - if '_AF_' in simName: - nameParts = simName.split('_AF_') - infoParts = nameParts[1].split('_') + if "_AF_" in simName: + nameParts = simName.split("_AF_") + infoParts = nameParts[1].split("_") else: - nameParts = simName.split('_') + nameParts = simName.split("_") infoParts = nameParts[1:] simHash = infoParts[0] cfgObject = readCfgFile(avaDir, fileName=cFile) @@ -533,9 +547,9 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS simDF = convertDF2numerics(simDF) # add default configuration - if standardCfg != '': + if standardCfg != "": # read default configuration of this module - simDF = appendCgf2DF('current standard', 'current standard', standardCfg, simDF) + simDF = appendCgf2DF("current standard", "current standard", standardCfg, simDF) # if writeCSV, write dataFrame to csv file if writeCSV: @@ -545,31 +559,31 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS def appendCgf2DF(simHash, simName, cfgObject, simDF): - """ append simulation configuration to the simulation dataframe - only account for sections GENERAL and INPUT - - Parameters - ----------- - simHash: str - hash of the simulation to append - simName: str - name of the simulation - cfgObject: configParser - configuration coresponding to the simulation - simDF: pandas dataFrame - configuration dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append simulation configuration to the simulation dataframe + only account for sections GENERAL and INPUT + + Parameters + ----------- + simHash: str + hash of the simulation to append + simName: str + name of the simulation + cfgObject: configParser + configuration coresponding to the simulation + simDF: pandas dataFrame + configuration dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] cfgDict = convertConfigParserToDict(cfgObject) - simItemDFGeneral = pd.DataFrame(data=cfgDict['GENERAL'], index=indexItem) - simItemDFInput = pd.DataFrame(data=cfgDict['INPUT'], index=indexItem) - if 'VISUALISATION' in cfgDict: - simItemDFVisualisation = pd.DataFrame(data=cfgDict['VISUALISATION'], index=indexItem) + simItemDFGeneral = pd.DataFrame(data=cfgDict["GENERAL"], index=indexItem) + simItemDFInput = pd.DataFrame(data=cfgDict["INPUT"], index=indexItem) + if "VISUALISATION" in cfgDict: + simItemDFVisualisation = pd.DataFrame(data=cfgDict["VISUALISATION"], index=indexItem) simItemDF = pd.concat([simItemDFGeneral, simItemDFInput, simItemDFVisualisation], axis=1) else: simItemDF = pd.concat([simItemDFGeneral, simItemDFInput], axis=1) @@ -582,21 +596,21 @@ def appendCgf2DF(simHash, simName, cfgObject, simDF): def appendTcpu2DF(simHash, tCPU, tCPUDF): - """ append Tcpu dictionary to the dataframe - - Parameters - ----------- - simHash: str - hash of the simulation corresponding to the tCPU dict to append - tCPU: dict - cpu time dict of the simulation - tCPUDF: pandas dataFrame - tCPU dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append Tcpu dictionary to the dataframe + + Parameters + ----------- + simHash: str + hash of the simulation corresponding to the tCPU dict to append + tCPU: dict + cpu time dict of the simulation + tCPUDF: pandas dataFrame + tCPU dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] tCPUItemDF = pd.DataFrame(data=tCPU, index=indexItem) @@ -608,99 +622,99 @@ def appendTcpu2DF(simHash, tCPU, tCPUDF): def convertDF2numerics(simDF): - """ convert a string DF to a numerical one + """convert a string DF to a numerical one - Parameters - ----------- - simDF: pandas dataFrame - dataframe + Parameters + ----------- + simDF: pandas dataFrame + dataframe - Returns - -------- - simDF: pandas DataFrame + Returns + -------- + simDF: pandas DataFrame """ for name, values in simDF.items(): - simDFTest = simDF[name].str.replace('.', '', regex=False) + simDFTest = simDF[name].str.replace(".", "", regex=False) # allow for - sign too - simDFTest = simDFTest.replace('-', '', regex=False) + simDFTest = simDFTest.replace("-", "", regex=False) # check for str(np.nan) as these cannot be converted to numerics by pd.to_numeric # but as friction model parameters are set to nans this is required here - if simDFTest.str.match('nan').any(): + if simDFTest.str.match("nan").any(): simDF = setStrnanToNan(simDF, simDFTest, name) # also include columns where nan is in first row - so check for any row if simDFTest.str.isdigit().any() and (name != 'tSteps'): # problem here is that it finds even if not present in | although not in ini - simDFTest = simDF[name].str.replace('|', '§', regex=False) - if simDFTest.str.contains('§').any() == False: + simDFTest = simDF[name].str.replace("|", "§", regex=False) + if simDFTest.str.contains("§").any() == False: simDF[name] = pd.to_numeric(simDF[name]) - log.debug('Converted to numeric %s' % name) + log.debug("Converted to numeric %s" % name) else: - log.debug('Not converted to numeric: %s' % name) + log.debug("Not converted to numeric: %s" % name) return simDF def setStrnanToNan(simDF, simDFTest, name): - """ set pandas element to np.nan if it is a string nan - - Parameters - ----------- - simDF: pandas dataFrame - dataframe - simDFTest: pandas series - series of sim DF column named name - replaced "." with " " - name: str - name of pandas dataframe column - - Returns - -------- - simDF: pandas dataframe - updated pandas dataframe with np.nan values where string nan was + """set pandas element to np.nan if it is a string nan + + Parameters + ----------- + simDF: pandas dataFrame + dataframe + simDFTest: pandas series + series of sim DF column named name + replaced "." with " " + name: str + name of pandas dataframe column + + Returns + -------- + simDF: pandas dataframe + updated pandas dataframe with np.nan values where string nan was """ - nanIndex = simDFTest.str.match('nan', flags=re.IGNORECASE) + nanIndex = simDFTest.str.match("nan", flags=re.IGNORECASE) simIndex = simDF.index.values # loop over each row and use simDF.at to avoid copy vs view warning for index, nanInd in enumerate(nanIndex): if nanInd: simDF.at[simIndex[index], name] = np.nan - log.info('%s for index: %s set to numpy nan' % (name, index)) + log.info("%s for index: %s set to numpy nan" % (name, index)) return simDF -def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfigurations'): - """ Read allConfigurations.csv file as dataFrame from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - specDir: str - path to a directory where simulation configuration files can be found - optional - configCsvName: str - name of configuration csv file - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations - simDFName: array - simName column of the dataframe +def readAllConfigurationInfo(avaDir, specDir="", configCsvName="allConfigurations"): + """Read allConfigurations.csv file as dataFrame from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + specDir: str + path to a directory where simulation configuration files can be found - optional + configCsvName: str + name of configuration csv file + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations + simDFName: array + simName column of the dataframe """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') - configFiles = inDir / ('%s.csv' % configCsvName) + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") + configFiles = inDir / ("%s.csv" % configCsvName) if configFiles.is_file(): - with open(configFiles, 'rb') as file: + with open(configFiles, "rb") as file: simDF = pd.read_csv(file, index_col=0, keep_default_na=False) - simDFName = simDF['simName'].to_numpy() + simDFName = simDF["simName"].to_numpy() else: simDF = None simDFName = [] @@ -708,31 +722,31 @@ def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfiguration return simDF, simDFName -def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurations.csv'): - """ Write cfg configuration to allConfigurations.csv - - Parameters - ----------- - avaDir: str - path to avalanche directory - simDF: pandas dataFrame - daaframe of the configuration - specDir: str - path to a directory where simulation configuration shal be saved - optional - csvName: str - name of csv file in which to save to - optional - - Returns - -------- - configFiles: pathlib Path - path where the configuration dataframe was saved +def writeAllConfigurationInfo(avaDir, simDF, specDir="", csvName="allConfigurations.csv"): + """Write cfg configuration to allConfigurations.csv + + Parameters + ----------- + avaDir: str + path to avalanche directory + simDF: pandas dataFrame + daaframe of the configuration + specDir: str + path to a directory where simulation configuration shal be saved - optional + csvName: str + name of csv file in which to save to - optional + + Returns + -------- + configFiles: pathlib Path + path where the configuration dataframe was saved """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") configFiles = inDir / csvName simDF.to_csv(configFiles) @@ -743,51 +757,51 @@ def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurati def convertToCfgList(parameterList): """ convert a list into a string where individual list items are separated by | - Parameters - ----------- - parameterList: list - list of parameter values + Parameters + ----------- + parameterList: list + list of parameter values - Returns - --------- - parameterString: str - str with parameter values separated by | + Returns + --------- + parameterString: str + str with parameter values separated by | """ if len(parameterList) == 0: - parameterString = '' + parameterString = "" else: parameterString = parameterList[0] for item in parameterList[1:]: - parameterString = parameterString + '|' + item + parameterString = parameterString + "|" + item return parameterString def getNumberOfProcesses(cfgMain, nSims): - """ Determine how many CPU cores to take for parallel tasks + """Determine how many CPU cores to take for parallel tasks - Parameters - ----------- - cfgMain: configuration object - the main avaframe configuration - nSims: integer - number of simulations that need to be calculated + Parameters + ----------- + cfgMain: configuration object + the main avaframe configuration + nSims: integer + number of simulations that need to be calculated - Returns - --------- - nCPU: int - number of cores to take + Returns + --------- + nCPU: int + number of cores to take """ maxCPU = multiprocessing.cpu_count() - if cfgMain["MAIN"]["nCPU"] == 'auto': - cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100. + if cfgMain["MAIN"]["nCPU"] == "auto": + cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100.0 nCPU = math.floor(maxCPU * cpuPerc) else: - nCPU = cfgMain['MAIN'].getint('nCPU') + nCPU = cfgMain["MAIN"].getint("nCPU") # if number of sims is lower than nCPU nCPU = min(nCPU, nSims) diff --git a/avaframe/out3Plot/plotUtils.py b/avaframe/out3Plot/plotUtils.py index b94f06400..cb4f36a6f 100644 --- a/avaframe/out3Plot/plotUtils.py +++ b/avaframe/out3Plot/plotUtils.py @@ -36,7 +36,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module -cfg = cfgUtils.getModuleConfig(plotUtils) +cfg = cfgUtils.getModuleConfig(plotUtils, cfgMain['MAIN']['configurationDir']) cfgPlotUtils = cfg["UNITS"] cfgConstants = cfg["CONSTANTS"] cfg = cfg["MAIN"] diff --git a/avaframe/runCom1DFA.py b/avaframe/runCom1DFA.py index 4d72def6e..f64642d58 100644 --- a/avaframe/runCom1DFA.py +++ b/avaframe/runCom1DFA.py @@ -17,8 +17,8 @@ from avaframe.in3Utils import fileHandlerUtils as fU -def runCom1DFA(avalancheDir='', calibration=''): - """ Run com1DFA in the default configuration with only an +def runCom1DFA(avalancheDir="", configurationDir="", calibration=""): + """Run com1DFA in the default configuration with only an avalanche directory as input and the (optional) friction calibration size @@ -39,21 +39,25 @@ def runCom1DFA(avalancheDir='', calibration=''): startTime = time.time() # log file name; leave empty to use default runLog.log - logName = 'runCom1DFA' + logName = "runCom1DFA" # Load avalanche directory from general configuration file # More information about the configuration can be found here # on the Configuration page in the documentation cfgMain = cfgUtils.getGeneralConfig() - if avalancheDir != '': - cfgMain['MAIN']['avalancheDir'] = avalancheDir + if avalancheDir != "": + cfgMain["MAIN"]["avalancheDir"] = avalancheDir else: - avalancheDir = cfgMain['MAIN']['avalancheDir'] + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + if configurationDir != "": + cfgMain["MAIN"]["configurationDir"] = configurationDir + else: + configurationDir = cfgMain["MAIN"]["configurationDir"] # Start logging log = logUtils.initiateLogger(avalancheDir, logName) - log.info('MAIN SCRIPT') - log.info('Current avalanche: %s', avalancheDir) + log.info("MAIN SCRIPT") + log.info("Current avalanche: %s", avalancheDir) # ---------------- # Clean input directory(ies) of old work and output files @@ -62,18 +66,18 @@ def runCom1DFA(avalancheDir='', calibration=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Set friction model according to cmd argument - cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, toPrint=False) - - if calibration.lower() == 'small': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATSmall' - elif calibration.lower() == 'medium': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATMedium' - elif calibration.lower() == 'large': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosAT' - elif calibration.lower() == 'auto': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATAuto' + cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, configDir=configurationDir, toPrint=False) + + if calibration.lower() == "small": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATSmall" + elif calibration.lower() == "medium": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATMedium" + elif calibration.lower() == "large": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosAT" + elif calibration.lower() == "auto": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATAuto" else: - log.info('no friction calibration override given - using ini') + log.info("no friction calibration override given - using ini") # ---------------- # Run dense flow @@ -81,25 +85,34 @@ def runCom1DFA(avalancheDir='', calibration=''): # Get peakfiles to return to QGIS avaDir = pathlib.Path(avalancheDir) - inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' + inputDir = avaDir / "Outputs" / "com1DFA" / "peakFiles" peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) # Print time needed endTime = time.time() - log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + log.info("Took %6.1f seconds to calculate." % (endTime - startTime)) return peakFilesDF -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description='Run com1DFA workflow') - parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', - help='the avalanche directory') - parser.add_argument('-fc', '--friction_calibration', choices=['auto', 'large', 'medium', 'small', 'ini'], - type=str, default='ini', - help='friction calibration override, possible values are large, medium , small, auto and ini.' + - 'Overrides default AND local configs') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run com1DFA workflow") + parser.add_argument( + "avadir", metavar="avalancheDir", type=str, nargs="?", default="", help="the avalanche directory" + ) + parser.add_argument( + "configdir", metavar="configurationDir", type=str, nargs="?", default="", + help="the configuration file directory" + ) + parser.add_argument( + "-fc", + "--friction_calibration", + choices=["auto", "large", "medium", "small", "ini"], + type=str, + default="ini", + help="friction calibration override, possible values are large, medium , small, auto and ini." + + "Overrides default AND local configs", + ) args = parser.parse_args() - runCom1DFA(str(args.avadir), str(args.friction_calibration)) + runCom1DFA(str(args.avadir), str(args.configdir),str(args.friction_calibration)) diff --git a/avaframe/runScripts/runPlotContoursFromAsc.py b/avaframe/runScripts/runPlotContoursFromAsc.py index 04dac701f..46911606b 100644 --- a/avaframe/runScripts/runPlotContoursFromAsc.py +++ b/avaframe/runScripts/runPlotContoursFromAsc.py @@ -26,7 +26,7 @@ def plotContoursFromAsc(cfg, avalancheDir): if __name__ == "__main__": # Load configuration for runPlotContour - cfg = cfgUtils.getModuleConfig(rCon, fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) + cfg = cfgUtils.getModuleConfig(rCon, '', fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) # fetch input directory cfgMain = cfgUtils.getGeneralConfig() diff --git a/docs/configuration.rst b/docs/configuration.rst index 402662068..033aeeda0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -61,4 +61,14 @@ in the module **B** configuration file are used to update the configuration sett the configuration parameters used for one task in one configuration file. An example of this usage can be found in ``ana1Tests/energyLineTestCfg.ini``. +Additionally, there is the option to write ``local_moduleNameCfg.ini`` configuration files for all modules +where a corresponding *collectionName_moduleName_override* section is provided by using +the function: :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`. These ``local_moduleNameCfg.ini`` files are +saved to ``avalancheDir/Inputs/configurationsOverrides`` by default if no ``localCfgPath`` is provided as input. +An example of this functionality is provided in :py:mod:`runScripts.runPlotContoursFromAsc.py` with the respective +``runPlotContoursFromAscCfg.ini`` file. + +.. Note:: + By using :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`, already existing ``local_moduleNameCfg.ini`` files in the + respective directory will be overwritten.