From 670db2c1b85de686f065c801ce9b84ade644fb49 Mon Sep 17 00:00:00 2001 From: Chris Weber Date: Wed, 1 Apr 2020 19:06:30 +0200 Subject: [PATCH 01/18] WIP: Adds logging --- src/spotpy/algorithms/_algorithm.py | 86 ++++++++++++++------------- src/spotpy/algorithms/abc.py | 27 ++++----- src/spotpy/algorithms/dds.py | 22 ++----- src/spotpy/algorithms/demcz.py | 23 +++---- src/spotpy/algorithms/fast.py | 21 ++++++- src/spotpy/algorithms/fscabc.py | 20 +++---- src/spotpy/algorithms/lhs.py | 4 +- src/spotpy/algorithms/list_sampler.py | 2 +- src/spotpy/algorithms/mc.py | 2 +- src/spotpy/algorithms/mcmc.py | 12 ++-- src/spotpy/algorithms/mle.py | 8 +-- src/spotpy/algorithms/padds.py | 6 +- src/spotpy/algorithms/rope.py | 29 ++++----- src/spotpy/algorithms/sa.py | 13 ++-- src/spotpy/algorithms/sceua.py | 55 ++++++++--------- src/spotpy/database/base.py | 17 ++++-- src/spotpy/database/csv.py | 4 +- src/spotpy/database/hdf5.py | 4 +- src/spotpy/objectivefunctions.py | 44 +++++++------- src/spotpy/spotpylogging.py | 34 +++++++++++ 20 files changed, 234 insertions(+), 199 deletions(-) create mode 100644 src/spotpy/spotpylogging.py diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index 22890910..847c99f8 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -12,6 +12,7 @@ import numpy as np from spotpy import database, parameter +from spotpy import spotpylogging try: from queue import Queue @@ -35,17 +36,18 @@ class _RunStatistic(object): """ def __init__(self, repetitions, algorithm_name, optimization_direction, parnames): - self.optimization_direction = optimization_direction # grid, mazimize, minimize - print( - "Initializing the ", algorithm_name, " with ", repetitions, " repetitions" - ) - if optimization_direction == "minimize": + self.optimization_direction = optimization_direction #grid, mazimize, minimize + + self.logger = spotpylogging.get_logger("RunStatistic(%s)" % algorithm_name ) + + self.logger.info('Initializing the %s with %s repetitions', algorithm_name, repetitions) + if optimization_direction == 'minimize': self.compare = self.minimizer - print("The objective function will be minimized") - if optimization_direction == "maximize": + self.logger.info('The objective function will be minimized') + if optimization_direction == 'maximize': self.compare = self.maximizer - print("The objective function will be maximized") - if optimization_direction == "grid": + self.logger.info('The objective function will be maxiimized') + if optimization_direction == 'grid': self.compare = self.grid self.rep = 0 @@ -131,46 +133,43 @@ def print_status(self): timestr, ) - print(text) + self.logger.info(text) self.last_print = time.time() def print_status_final(self): - print("\n*** Final SPOTPY summary ***") - print( - "Total Duration: " - + str(round((time.time() - self.starttime), 2)) - + " seconds" - ) - print("Total Repetitions:", self.rep) + self.logger.info('\n*** Final SPOTPY summary ***') + self.logger.info('Total Duration: %s seconds' % str(round((time.time() - self.starttime), 2))) + self.logger.info('Total Repetitions: %s', self.rep) - if self.optimization_direction == "minimize": - print("Minimal objective value: %g" % (self.objectivefunction_min)) - print("Corresponding parameter setting:") + if self.optimization_direction == 'minimize': + self.logger.info('Minimal objective value: %g' % (self.objectivefunction_min)) + self.logger.info('Corresponding parameter setting:') for i in range(self.parameters): - text = "%s: %g" % (self.parnames[i], self.params_min[i]) - print(text) + text = '%s: %g' % (self.parnames[i], self.params_min[i]) + self.logger.info(text) - if self.optimization_direction == "maximize": - print("Maximal objective value: %g" % (self.objectivefunction_max)) - print("Corresponding parameter setting:") + if self.optimization_direction == 'maximize': + self.logger.info('Maximal objective value: %g' % (self.objectivefunction_max)) + self.logger.info('Corresponding parameter setting:') for i in range(self.parameters): - text = "%s: %g" % (self.parnames[i], self.params_max[i]) - print(text) + text = '%s: %g' % (self.parnames[i], self.params_max[i]) + self.logger.info(text) - if self.optimization_direction == "grid": - print("Minimal objective value: %g" % (self.objectivefunction_min)) - print("Corresponding parameter setting:") + if self.optimization_direction == 'grid': + self.logger.info('Minimal objective value: %g' % (self.objectivefunction_min)) + self.logger.info('Corresponding parameter setting:') for i in range(self.parameters): - text = "%s: %g" % (self.parnames[i], self.params_min[i]) - print(text) + text = '%s: %g' % (self.parnames[i], self.params_min[i]) + self.logger.info(text) - print("Maximal objective value: %g" % (self.objectivefunction_max)) - print("Corresponding parameter setting:") + self.logger.info('Maximal objective value: %g' % (self.objectivefunction_max)) + self.logger.info('Corresponding parameter setting:') for i in range(self.parameters): - text = "%s: %g" % (self.parnames[i], self.params_max[i]) - print(text) + text = '%s: %g' % (self.parnames[i], self.params_max[i]) + self.logger.info(text) + + self.logger.info('******************************\n') - print("******************************\n") def __repr__(self): return "Min objectivefunction: %g \n Max objectivefunction: %g" % ( @@ -243,6 +242,9 @@ def __init__( algorithm_name="", ): + # Instatiate logging + self.logger = spotpylogging.get_logger(self.__class__.__name__) + # Initialize the user defined setup class self.setup = spot_setup param_info = parameter.get_parameters_array( @@ -292,11 +294,11 @@ def __init__( self._return_all_likes = False # allows multi-objective calibration if set to True, is set by the algorithm if breakpoint == "read" or breakpoint == "readandwrite": - print("Reading backupfile") + self.logger.info('Reading backupfile') try: open(self.dbname + ".break") except FileNotFoundError: - print("Backupfile not found") + self.logger.info('Backupfile not found') self.dbappend = True # Now a repeater (ForEach-object) is loaded @@ -370,7 +372,7 @@ def final_call(self): def _init_database(self, like, randompar, simulations): if self.dbinit: - print("Initialize database...") + self.logger.info('Initialize database...') self.datawriter = database.get_datawriter( self.dbformat, @@ -494,7 +496,7 @@ def getfitness(self, simulation, params): Calls the user defined spot_setup objectivefunction """ try: - # print('Using parameters in fitness function') + # self.logger.info('Using parameters in fitness function') return self.setup.objectivefunction( evaluation=self.evaluation, simulation=simulation, @@ -502,7 +504,7 @@ def getfitness(self, simulation, params): ) except TypeError: # Happens if the user does not allow to pass parameter in the spot_setup.objectivefunction - # print('Not using parameters in fitness function') + # self.logger.info('Not using parameters in fitness function') return self.setup.objectivefunction( evaluation=self.evaluation, simulation=simulation ) diff --git a/src/spotpy/algorithms/abc.py b/src/spotpy/algorithms/abc.py index f0f67b85..e96499eb 100644 --- a/src/spotpy/algorithms/abc.py +++ b/src/spotpy/algorithms/abc.py @@ -78,9 +78,7 @@ def sample( sets the limit """ self.set_repetiton(repetitions) - print( - "Starting the ABC algotrithm with " + str(repetitions) + " repetitions..." - ) + self.logger.info('Starting the ABC algotrithm with %s repetitions...', repetitions) # Initialize ABC parameters: randompar = self.parameter()["random"] self.nopt = randompar.size @@ -106,7 +104,7 @@ def sample( work.append([like, randompar, like, randompar, c, p]) icall += 1 if self.status.stop: - print("Stopping sampling") + self.logger.info('Stopping sampling') break while icall < repetitions and gnrng > peps: @@ -141,7 +139,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info('Stopping samplig') break # Probability distribution for roulette wheel selection bn = [] for i, val in enumerate(work): @@ -191,7 +189,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info('Stopping samplig') break # Scout bee phase for i, val in enumerate(work): @@ -205,18 +203,17 @@ def sample( work[i][0] = clike icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info('Stopping samplig') break gnrng = -self.status.objectivefunction_max if icall >= repetitions: - print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") - print("ON THE MAXIMUM NUMBER OF TRIALS ") - print(repetitions) - print("HAS BEEN EXCEEDED.") + self.logger.info('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') + self.logger.info('ON THE MAXIMUM NUMBER OF TRIALS ') + self.logger.info(repetitions) + self.logger.info('HAS BEEN EXCEEDED.') if gnrng < peps: - print( - "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN" - ) - print(icall) + self.logger.info( + 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN') + self.logger.info(icall) self.final_call() diff --git a/src/spotpy/algorithms/dds.py b/src/spotpy/algorithms/dds.py index fe5c31d0..c803db50 100644 --- a/src/spotpy/algorithms/dds.py +++ b/src/spotpy/algorithms/dds.py @@ -281,9 +281,7 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.parameter()["minbound"], self.parameter()["maxbound"], ) - print( - "Starting the DDS algotrithm with " + str(repetitions) + " repetitions..." - ) + self.logger.info('Starting the DDS algotrithm with %s repetitions...', repetitions) number_of_parameters = ( self.status.parameters @@ -334,13 +332,8 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.params_max = list(x_curr) self.params_max = self.fix_status_params_format(self.params_max) - print( - "Best solution found has obj function value of " - + str(objectivefunction_max) - + " at " - + str(repitionno_best) - + "\n\n" - ) + self.logger.info('Best solution found has obj function value of %s at %s\n\n', \ + objectivefunction_max, repitionno_best) debug_results.append( { "sbest": self.params_max, @@ -370,13 +363,8 @@ def calc_initial_para_configuration( # by trying which randomized generated input matches best # initial_iterations is the number of function evaluations to initialize the DDS algorithm solution if initial_iterations > 1: - print( - "Finding best starting point for trial " - + str(trial + 1) - + " using " - + str(initial_iterations) - + " random samples." - ) + self.logger.info("Finding best starting point for trial %s using %s random samples.", \ + trial + 1, initial_iterations) repetions_left = ( repetitions - initial_iterations ) # use this to reduce number of fevals in DDS loop diff --git a/src/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py index b3fef8fa..d785df59 100644 --- a/src/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -8,6 +8,7 @@ import numpy as np from . import _algorithm +from spotpy import spotpylogging class demcz(_algorithm): @@ -104,7 +105,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print("ERROR Bounds have not the same lenghts as Parameterarray") + self.logger.info("ERROR Bounds have not the same lenghts as Parameterarray") return par def sample( @@ -143,8 +144,9 @@ def sample( """ self.set_repetiton(repetitions) - print( - "Starting the DEMCz algotrithm with " + str(repetitions) + " repetitions..." + self.logger.info( + "Starting the DEMCz algotrithm with %s repetitions...", + repetitions ) self.min_bound, self.max_bound = ( @@ -220,9 +222,9 @@ def sample( # 3) and we have not done more than the maximum number of iterations while cur_iter < maxChainDraws: - print(cur_iter, burnIn) + self.logger.info("%s, %s", cur_iter, burnIn) if cur_iter == burnIn: - print("starting") + self.logger.info('starting') history.start_sampling() # every5th iteration allow a big jump @@ -349,7 +351,7 @@ def sample( covConvergence.update(history, "interest") if all(grConvergence.R < convergenceCriteria): cur_iter = maxChainDraws - print( + self.logger.info( "All chains fullfil the convergence criteria. Sampling stopped." ) cur_iter += 1 @@ -362,11 +364,12 @@ def sample( self.iter = cur_iter self.burnIn = burnIn self.R = grConvergence.R - text = "Gelman Rubin R=" + str(self.R) - print(text) + + self.logger.info('Gelman Rubin R=%s', self.R) self.status.rep = self.status.repetitions self.final_call() + def _update_accepts_ratio(self, weighting, acceptances): self.accepts_ratio = ( weighting * np.mean(acceptances) + (1 - weighting) * self.accepts_ratio @@ -561,9 +564,9 @@ def rv(relevantHistory): try: projection = np.dot(np.linalg.inv(basis1), basis2) - except np.linalg.linalg.LinAlgError: + except np.linalg.linalg.LinAlgError as e: projection = np.array(basis1) * np.nan - print("Exception happend!") + spotpylogging.get_logger("_CovarianceConvergence()").logger.info("Exception happend!\nExcpetion:%s", e) # find the releative size in each of the basis1 directions return np.log(np.sum(projection**2, axis=0) ** 0.5) diff --git a/src/spotpy/algorithms/fast.py b/src/spotpy/algorithms/fast.py index 143e369a..5a404d21 100644 --- a/src/spotpy/algorithms/fast.py +++ b/src/spotpy/algorithms/fast.py @@ -131,14 +131,14 @@ def matrix(self, bounds, N, M=4): def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): if len(Y.shape) > 1: Y = Y.flatten() - print(Y.size) + self.logger.info(Y.size) if Y.size % (D) == 0: N = int(Y.size / D) elif Y.size > D: N = int(Y.size / D) rest = Y.size - N * D - print( + self.logger.info( """ We can not use """ + str(rest) @@ -149,7 +149,7 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): """ ) else: - print( + self.logger.info( """ Error: Number of samples in model output file must be a multiple of D, where D is the number of parameters in your parameter file. @@ -169,14 +169,24 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): # Calculate and Output the First and Total Order Values if print_to_console: +<<<<<<< HEAD:src/spotpy/algorithms/fast.py print("Parameter First Total") Si = dict((k, [None] * D) for k in ["S1", "ST"]) +======= + self.logger.info("Parameter First Total") + Si = dict((k, [None] * D) for k in ['S1', 'ST']) +>>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py for i in range(D): l = np.arange(i * N, (i + 1) * N) Si["S1"][i] = self.compute_first_order(Y[l], N, M, omega[0]) Si["ST"][i] = self.compute_total_order(Y[l], N, omega[0]) if print_to_console: +<<<<<<< HEAD:src/spotpy/algorithms/fast.py print("%s %f %f" % (parnames[i], Si["S1"][i], Si["ST"][i])) +======= + self.logger.info("%s %f %f" % + (parnames[i], Si['S1'][i], Si['ST'][i])) +>>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py return Si def compute_first_order(self, outputs, N, M, omega): @@ -203,10 +213,15 @@ def sample(self, repetitions, M=4): Maximum number of runs. """ self.set_repetiton(repetitions) +<<<<<<< HEAD:src/spotpy/algorithms/fast.py print( "Starting the FAST algotrithm with " + str(repetitions) + " repetitions..." ) print("Creating FAST Matrix") +======= + self.logger.info('Starting the FAST algotrithm with '+str(repetitions)+ ' repetitions...') + self.logger.info('Creating FAST Matrix') +>>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py # Get the names of the parameters to analyse names = self.parameter()["name"] # Get the minimum and maximum value for each parameter from the diff --git a/src/spotpy/algorithms/fscabc.py b/src/spotpy/algorithms/fscabc.py index 03aec53a..318b1a23 100644 --- a/src/spotpy/algorithms/fscabc.py +++ b/src/spotpy/algorithms/fscabc.py @@ -86,7 +86,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None sets the limit for scout bee phase """ self.set_repetiton(repetitions) - print( + self.logger.info( "Starting the FSCABC algotrithm with " + str(repetitions) + " repetitions..." @@ -134,7 +134,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None icall += 1 if self.status.stop: # icall = repetitions - print("Stopping samplig") + self.logger.info("Stopping sampling") break # Bee Phases @@ -174,7 +174,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info("Stopping sampling") break # Fitness scaling @@ -223,7 +223,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info("Stopping samplig") break # Scout bee phase for i, val in enumerate(work): @@ -239,7 +239,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[i][0] = clike icall += 1 if self.status.stop: - print("Stopping samplig") + self.logger.info("Stopping samplig") break gnrng = -self.status.objectivefunction_max @@ -252,13 +252,13 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None self.write_breakdata(self.dbname, savework) lastbackup = icall if icall >= repetitions: - print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") - print("ON THE MAXIMUM NUMBER OF TRIALS ") - print(repetitions) - print("HAS BEEN EXCEEDED.") + self.logger.info("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.info("ON THE MAXIMUM NUMBER OF TRIALS ") + self.logger.info(repetitions) + self.logger.info("HAS BEEN EXCEEDED.") if gnrng < peps: - print( + self.logger.info( "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" ) self.final_call() diff --git a/src/spotpy/algorithms/lhs.py b/src/spotpy/algorithms/lhs.py index 998857fd..56ad2c24 100644 --- a/src/spotpy/algorithms/lhs.py +++ b/src/spotpy/algorithms/lhs.py @@ -61,10 +61,10 @@ def sample(self, repetitions): maximum number of function evaluations allowed during optimization """ self.set_repetiton(repetitions) - print( + self.logger.info( "Starting the LHS algotrithm with " + str(repetitions) + " repetitions..." ) - print("Creating LatinHyperCube Matrix") + self.logger.info("Creating LatinHyperCube Matrix") # Get the names of the parameters to analyse names = self.parameter()["name"] # Define the jump size between the parameter diff --git a/src/spotpy/algorithms/list_sampler.py b/src/spotpy/algorithms/list_sampler.py index 8f386e2a..45ebbf02 100644 --- a/src/spotpy/algorithms/list_sampler.py +++ b/src/spotpy/algorithms/list_sampler.py @@ -68,7 +68,7 @@ def sample(self, repetitions=None): self.set_repetiton(repetitions) # Initialization - print("Starting the List sampler with " + str(repetitions) + " repetitions...") + self.logger.info("Starting the List sampler with " + str(repetitions) + " repetitions...") param_generator = ( (rep, list(parameters[rep])) for rep in range(int(repetitions)) ) diff --git a/src/spotpy/algorithms/mc.py b/src/spotpy/algorithms/mc.py index 171ac155..ddd71440 100644 --- a/src/spotpy/algorithms/mc.py +++ b/src/spotpy/algorithms/mc.py @@ -61,7 +61,7 @@ def sample(self, repetitions): Maximum number of runs. """ self.set_repetiton(repetitions) - print("Starting the MC algorithm with {} repetitions...".format(repetitions)) + self.logger.info("Starting the MC algorithm with %s repetitions...", repetitions) # A generator that produces parametersets if called param_generator = ( (rep, self.parameter()["random"]) for rep in range(int(repetitions)) diff --git a/src/spotpy/algorithms/mcmc.py b/src/spotpy/algorithms/mcmc.py index 04078c68..6c2481ba 100644 --- a/src/spotpy/algorithms/mcmc.py +++ b/src/spotpy/algorithms/mcmc.py @@ -63,7 +63,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print("ERROR: Bounds have not the same lenghts as Parameterarray") + self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") return par def check_par_validity_reflect(self, par): @@ -81,7 +81,7 @@ def check_par_validity_reflect(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print("ERROR: Bounds have not the same lenghts as Parameterarray") + self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") return par def get_new_proposal_vector(self, old_par): @@ -97,7 +97,7 @@ def update_mcmc_status(self, par, like, sim, cur_chain): def sample(self, repetitions, nChains=1): self.set_repetiton(repetitions) - print( + self.logger.info( "Starting the MCMC algotrithm with " + str(repetitions) + " repetitions..." ) # Prepare storing MCMC chain as array of arrays. @@ -117,7 +117,7 @@ def sample(self, repetitions, nChains=1): self.parameter()["minbound"], self.parameter()["maxbound"], ) - print("Initialize ", self.nChains, " chain(s)...") + self.logger.info("Initialize ", self.nChains, " chain(s)...") self.iter = 0 param_generator = ( (curChain, self.parameter()["random"]) @@ -132,7 +132,7 @@ def sample(self, repetitions, nChains=1): self.iter += 1 intervaltime = time.time() - print("Beginn of Random Walk") + self.logger.info("Beginn of Random Walk") # Walk through chains while self.iter <= repetitions - self.burnIn: param_generator = ( @@ -167,6 +167,6 @@ def sample(self, repetitions, nChains=1): ) * 100 ).strip("array([])") - print(text) + self.logger.info(text) intervaltime = time.time() self.final_call() diff --git a/src/spotpy/algorithms/mle.py b/src/spotpy/algorithms/mle.py index 62bc1a82..f481c825 100644 --- a/src/spotpy/algorithms/mle.py +++ b/src/spotpy/algorithms/mle.py @@ -59,12 +59,12 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print("ERROR Bounds have not the same lenghts as Parameterarray") + self.logger.info("ERROR Bounds have not the same lenghts as Parameterarray") return par def sample(self, repetitions): self.set_repetiton(repetitions) - print( + self.logger.info( "Starting the MLE algotrithm with " + str(repetitions) + " repetitions..." ) # Define stepsize of MLE @@ -79,7 +79,7 @@ def sample(self, repetitions): likes = [] pars = [] sims = [] - print("burnIn...") + self.logger.info("burnIn...") for i in range(burnIn): randompar = self.parameter()["random"] pars.append(randompar) @@ -90,7 +90,7 @@ def sample(self, repetitions): old_like = max(likes) old_par = pars[likes.index(old_like)] - print("Beginn Random Walk") + self.logger.info("Beginn Random Walk") for rep in range(repetitions - burnIn): # Suggest new candidate from Gaussian proposal distribution. # Use stepsize provided for every dimension. diff --git a/src/spotpy/algorithms/padds.py b/src/spotpy/algorithms/padds.py index ce35b7e4..ee32d3c2 100644 --- a/src/spotpy/algorithms/padds.py +++ b/src/spotpy/algorithms/padds.py @@ -231,10 +231,10 @@ def sample( # every iteration a map of all relevant values is stored, only for debug purpose. # Spotpy will not need this values. debug_results = [] - print( + self.logger.info( "Starting the PADDS algotrithm with " + str(repetitions) + " repetitions..." ) - print( + self.logger.info( "WARNING: THE PADDS algorithm as implemented in SPOTPY is in an beta stage and not ready for production use!" ) self.set_repetiton(repetitions) @@ -292,7 +292,7 @@ def sample( self.parameter_current, ) - print( + self.logger.info( "Best solution found has obj function value of " + str(self.best_value.best_obj_val) + " at " diff --git a/src/spotpy/algorithms/rope.py b/src/spotpy/algorithms/rope.py index 88f60224..77b5bef8 100644 --- a/src/spotpy/algorithms/rope.py +++ b/src/spotpy/algorithms/rope.py @@ -106,7 +106,7 @@ def sample( # Takes ways´to long for npar >8 # wenn mehr parameter produziert werden sollen als reingehen, rechnet er sich tot (ngen>n) # Subsets < 5 führt manchmal zu Absturz - print( + self.logger.info( "Starting the ROPE algotrithm with " + str(repetitions) + " repetitions..." ) self.set_repetiton(repetitions) @@ -122,11 +122,9 @@ def sample( repetitions_following_runs = int((repetitions - first_run) / (subsets - 1)) # Needed to avoid an error in integer division somewhere in depth function if repetitions_following_runs % 2 != 0: - print( - "Warning: Burn-in samples and total number of repetions are not compatible.\n" - "SPOTPY will automatically adjust the number of total repetitions." - ) - repetitions_following_runs += 1 + self.logger.info('Warning: Burn-in samples and total number of repetions are not compatible.\n' + 'SPOTPY will automatically adjust the number of total repetitions.') + repetitions_following_runs+=1 if NDIR is None: NDIR = int(repetitions_following_runs / 100.0) @@ -167,12 +165,9 @@ def sample( if self.status.stop: break if acttime - intervaltime >= 2: - text = "1 Subset: Run %i of %i (best like=%g)" % ( - rep, - first_run, - self.status.objectivefunction_max, - ) - print(text) + text = '1 Subset: Run %i of %i (best like=%g)' % ( + rep, first_run, self.status.objectivefunction_max) + self.logger.info(text) intervaltime = time.time() for subset in range(subsets - 1): @@ -209,7 +204,7 @@ def sample( likes.append(like) pars.append(ropepar) if self.status.stop: - print("Stopping samplig") + self.logger.info('Stopping sampling') break # Progress bar @@ -223,7 +218,7 @@ def sample( repetitions_following_runs, self.status.objectivefunction_max, ) - print(text) + self.logger.info(text) intervaltime = time.time() if self.status.stop: break @@ -235,10 +230,10 @@ def programm_depth(self, pars, runs): N, NP = X.shape text = str(N) + " input vectors with " + str(NP) + " parameters" - print(text) + self.logger.info(text) Ngen = int(runs) # int(N*(1/self.percentage)) - print(("Generating " + str(Ngen) + " parameters:")) + self.logger.info(("Generating " + str(Ngen) + " parameters:")) NPOSI = Ngen # Number of points to generate @@ -270,7 +265,7 @@ def programm_depth(self, pars, runs): if LNDEP[L] >= 1: CL = np.vstack((CL, TL[L, :])) IPOS = IPOS + 1 - print((IPOS, ITRY)) + self.logger.info((IPOS, ITRY)) CL = np.delete(CL, 0, 0) CL = CL[:NPOSI] diff --git a/src/spotpy/algorithms/sa.py b/src/spotpy/algorithms/sa.py index ba787fae..fcf0499e 100644 --- a/src/spotpy/algorithms/sa.py +++ b/src/spotpy/algorithms/sa.py @@ -62,7 +62,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - print("ERROR: Bounds have not the same lenghts as Parameterarray") + self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") return par def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): @@ -75,12 +75,11 @@ def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): Maximum number of runs. """ self.set_repetiton(repetitions) - print("Starting the SA algotrithm with " + str(repetitions) + " repetitions...") - self.min_bound, self.max_bound = ( - self.parameter()["minbound"], - self.parameter()["maxbound"], - ) - stepsizes = self.parameter()["step"] + self.logger.info( + 'Starting the SA algotrithm with '+str(repetitions)+ ' repetitions...' + ) + self.min_bound, self.max_bound = self.parameter()['minbound'], self.parameter()['maxbound'] + stepsizes = self.parameter()['step'] Eopt = 999999 Titer = Tini x = self.parameter()["optguess"] diff --git a/src/spotpy/algorithms/sceua.py b/src/spotpy/algorithms/sceua.py index a0841b5c..f268e89c 100644 --- a/src/spotpy/algorithms/sceua.py +++ b/src/spotpy/algorithms/sceua.py @@ -206,7 +206,7 @@ def sample( icall = 0 xf = np.zeros(npt) - print("Starting burn-in sampling...") + self.logger.info('Starting burn-in sampling...') # Burn in param_generator = ((rep, x[rep]) for rep in range(int(npt))) @@ -216,9 +216,7 @@ def sample( xf[rep] = like icall += 1 if self.status.stop: - print( - "Stopping samplig. Maximum number of repetitions reached already during burn-in" - ) + self.logger.info('Stopping sampling. Maximum number of repetitions reached already during burn-in') proceed = False break # Sort the population in order of increasing function values; @@ -258,17 +256,16 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") - print("ON THE MAXIMUM NUMBER OF TRIALS ") - print(repetitions) - print("HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:") - print(self.status.rep) - print("OF THE INITIAL LOOP!") + self.logger.info('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') + self.logger.info('ON THE MAXIMUM NUMBER OF TRIALS ') + self.logger.info(repetitions) + self.logger.info('HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:') + self.logger.info(self.status.rep) + self.logger.info('OF THE INITIAL LOOP!') if gnrng < peps: - print( - "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" - ) + self.logger.info( + 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') # Begin evolution loops: nloop = 0 @@ -297,7 +294,7 @@ def sample( and proceed == True ): nloop += 1 - print("ComplexEvo loop #%d in progress..." % nloop) + self.logger.info('ComplexEvo loop #%d in progress...' % nloop) # Loop on complexes (sub-populations); cx = np.zeros((self.npg, self.nopt)) cf = np.zeros((self.npg)) @@ -336,7 +333,7 @@ def sample( i, pars[i], sims[i], chains=i + 1, save_run=False ) self.discarded_runs += 1 - print("Skipping saving") + self.logger.info("Skipping saving") if ( self.breakpoint == "write" @@ -378,20 +375,20 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - print("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") - print("ON THE MAXIMUM NUMBER OF TRIALS ") - print(repetitions) - print("HAS BEEN EXCEEDED.") + self.logger.info("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.info("ON THE MAXIMUM NUMBER OF TRIALS ") + self.logger.info(repetitions) + self.logger.info("HAS BEEN EXCEEDED.") elif gnrng < peps: - print( + self.logger.info( "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" ) elif ( nloop >= kstop ): # necessary so that the area of high posterior density is visited as much as possible - print( + self.logger.info( "Objective function convergence criteria is now being updated and assessed..." ) absolute_change = ( @@ -402,13 +399,13 @@ def sample( criter_change_pcent = 0.0 else: criter_change_pcent = absolute_change / denominator - print("Updated convergence criteria: %f" % criter_change_pcent) + self.logger.info("Updated convergence criteria: %f" % criter_change_pcent) if criter_change_pcent <= pcento: - print( + self.logger.info( "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY LESS THAN THE USER-SPECIFIED THRESHOLD %f" % (kstop, pcento) ) - print( + self.logger.info( "CONVERGENCY HAS ACHIEVED BASED ON OBJECTIVE FUNCTION CRITERIA!!!" ) elif self.status.stop: @@ -418,14 +415,14 @@ def sample( # stop, if max number of loop iteration was reached elif max_loop_inc and nloop >= max_loop_inc: proceed = False - print("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") + self.logger.info("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") break # End of the Outer Loops - print("SEARCH WAS STOPPED AT TRIAL NUMBER: %d" % self.status.rep) - print("NUMBER OF DISCARDED TRIALS: %d" % self.discarded_runs) - print("NORMALIZED GEOMETRIC RANGE = %f" % gnrng) - print( + self.logger.info("SEARCH WAS STOPPED AT TRIAL NUMBER: %d" % self.status.rep) + self.logger.info("NUMBER OF DISCARDED TRIALS: %d" % self.discarded_runs) + self.logger.info("NORMALIZED GEOMETRIC RANGE = %f" % gnrng) + self.logger.info( "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY %f PERCENT" % (kstop, criter_change_pcent) ) diff --git a/src/spotpy/database/base.py b/src/spotpy/database/base.py index 5e64b149..cd31b146 100644 --- a/src/spotpy/database/base.py +++ b/src/spotpy/database/base.py @@ -16,6 +16,8 @@ import numpy as np +from spotpy import spotpylogging + class database(object): """ @@ -36,6 +38,9 @@ def __init__( **kwargs, ): # Just needed for the first line in the database + + self.logger = spotpylogging.get_logger(self.__class__.__name__) + self.chains = chains self.dbname = dbname self.like = like @@ -87,15 +92,15 @@ def _empty_list(self, obj): return [] def _scalar_to_list(self, obj): - # print('scalar') + # self.logger.info('scalar') return [obj] def _iterable_to_list(self, obj): - # print('iterable') + # self.logger.info('iterable') return list(obj) def _array_to_list(self, obj): - # print('array') + # self.logger.info('array') values = [] for val in obj: values.append(val) @@ -103,13 +108,13 @@ def _array_to_list(self, obj): # return obj.flatten().tolist() def _nestediterable_to_list(self, obj): - # print('nested') + # self.logger.info('nested') values = [] for nestedlist in obj: - # print(len(nestedlist)) + # self.logger.info(len(nestedlist)) for val in nestedlist: values.append(val) - # print(len(values)) + # self.logger.info(len(values)) return values # return np.array(obj).flatten().tolist() diff --git a/src/spotpy/database/csv.py b/src/spotpy/database/csv.py index 008f1f34..a78bc54f 100644 --- a/src/spotpy/database/csv.py +++ b/src/spotpy/database/csv.py @@ -18,14 +18,14 @@ def __init__(self, *args, **kwargs): super(csv, self).__init__(*args, **kwargs) # store init item only if dbinit if kwargs.get("dbappend", False) is False: - print("* Database file '{}.csv' created.".format(self.dbname)) + self.logger.info("* Database file '{}.csv' created.".format(self.dbname)) # Create a open file, which needs to be closed after the sampling mode = "w" self.db = io.open(self.dbname + ".csv", mode) # write header line self.db.write(str(",".join(self.header) + "\n")) else: - print("* Appending to database file '{}.csv'.".format(self.dbname)) + self.logger.info("* Appending to database file '{}.csv'.".format(self.dbname)) # Continues writing file mode = "a" self.db = io.open(self.dbname + ".csv", mode) diff --git a/src/spotpy/database/hdf5.py b/src/spotpy/database/hdf5.py index a4a4153a..18cb54c8 100644 --- a/src/spotpy/database/hdf5.py +++ b/src/spotpy/database/hdf5.py @@ -14,10 +14,12 @@ from .base import database +from spotpy import spotpylogging + try: import tables except ImportError: - print( + spotpylogging.get_logger("hdf5").info( "ImportError: Pytables is not correctly installed. Please also make sure you", "installed the hdf5 extension (https://www.hdfgroup.org/downloads/hdf5/)", ) diff --git a/src/spotpy/objectivefunctions.py b/src/spotpy/objectivefunctions.py index 47de88ff..501d3391 100644 --- a/src/spotpy/objectivefunctions.py +++ b/src/spotpy/objectivefunctions.py @@ -9,13 +9,11 @@ This tool holds functions for statistic analysis. It takes Python-lists and returns the objective function value of interest. """ - -import logging - +from spotpy import spotpylogging import numpy as np +#logger.basicConfig(format='%(levelname)s: %(module)s.%(funcName)s(): %(message)s') -logging.basicConfig(format="%(levelname)s: %(module)s.%(funcName)s(): %(message)s") - +logger = spotpylogging.get_logger('objective_function') def bias(evaluation, simulation): """ @@ -41,7 +39,7 @@ def bias(evaluation, simulation): return float(bias) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -70,7 +68,7 @@ def pbias(evaluation, simulation): return 100 * (float(np.nansum(sim - obs)) / float(np.nansum(obs))) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -105,7 +103,7 @@ def nashsutcliffe(evaluation, simulation): return 1 - (numerator / denominator) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -140,7 +138,7 @@ def lognashsutcliffe(evaluation, simulation, epsilon=0): / sum((np.log(e) - np.mean(np.log(e))) ** 2) ) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -167,7 +165,7 @@ def log_p(evaluation, simulation): normpdf = -(y**2) / 2 - np.log(np.sqrt(2 * np.pi)) return np.mean(normpdf) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -194,7 +192,7 @@ def correlationcoefficient(evaluation, simulation): correlation_coefficient = np.corrcoef(evaluation, simulation)[0, 1] return correlation_coefficient else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -220,7 +218,7 @@ def rsquared(evaluation, simulation): if len(evaluation) == len(simulation): return correlationcoefficient(evaluation, simulation) ** 2 else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -249,7 +247,7 @@ def mse(evaluation, simulation): mse = np.nanmean((obs - sim) ** 2) return mse else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -275,7 +273,7 @@ def rmse(evaluation, simulation): if len(evaluation) == len(simulation) > 0: return np.sqrt(mse(evaluation, simulation)) else: - logging.warning("evaluation and simulation lists do not have the same length.") + logger.warning("evaluation and simulation lists do not have the same length.") return np.nan @@ -301,7 +299,7 @@ def mae(evaluation, simulation): mae = np.mean(np.abs(sim - obs)) return mae else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -329,7 +327,7 @@ def rrmse(evaluation, simulation): rrmse = rmse(evaluation, simulation) / np.mean(evaluation) return rrmse else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -366,7 +364,7 @@ def agreementindex(evaluation, simulation): ) return Agreement_index else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -395,7 +393,7 @@ def covariance(evaluation, simulation): covariance = np.mean((obs - obs_mean) * (sim - sim_mean)) return covariance else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -434,7 +432,7 @@ def decomposed_mse(evaluation, simulation): return decomposed_mse else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -464,7 +462,7 @@ def kge(evaluation, simulation, return_all=False): else: return kge else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -536,7 +534,7 @@ def kge_non_parametric(evaluation, simulation, return_all=False): else: return kge else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -556,7 +554,7 @@ def rsr(evaluation, simulation): rsr = rmse(evaluation, simulation) / np.std(evaluation) return rsr else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan @@ -589,7 +587,7 @@ def volume_error(evaluation, simulation): ve = np.sum(simulation - evaluation) / np.sum(evaluation) return float(ve) else: - logging.warning( + logger.warning( "evaluation and simulation lists does not have the same length." ) return np.nan diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py new file mode 100644 index 00000000..fc1178bc --- /dev/null +++ b/src/spotpy/spotpylogging.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +import logging + +# TODO possible next steps +# TODO 1. (tobias) remove logging calls with \n und passe die logging aufrufe an (grade alle info) +# TODO 2. (pr) make cli flags to control verbosity (ich weiß grade nicht wie das gehen soll) +# TODO 3. (pr) be able to create a file config + +path_to_logfile = 'spotpy.log' + +# Standard defintion of main logger +spotpy_logger = logging.getLogger("spotpy") + +# Add logging to stdout +handler_stdout = logging.StreamHandler() +formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') +handler_stdout.setFormatter(formatter) +spotpy_logger.addHandler(handler_stdout) + +# Add logging to file +handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments +handler_file.setFormatter(formatter) +spotpy_logger.addHandler(handler_file) + +spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments +spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) + + +def get_logger(name): + """ Returns a new child logger for the main spotpy application logging + + Use this logger to return new childs of the main logger""" + return spotpy_logger.getChild(name) # Set the logger name, with the implementation class From cf5eef3f45636469a5e7eefa58b03b35d2d83f7f Mon Sep 17 00:00:00 2001 From: Chris Weber Date: Thu, 2 Apr 2020 12:16:41 +0200 Subject: [PATCH 02/18] Removes todos --- src/spotpy/spotpylogging.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index fc1178bc..23429eef 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- +""" +Defines a standard logger for the main application, which every child can derive from. +Also it's possible to use the logger for the main application. +""" import logging -# TODO possible next steps -# TODO 1. (tobias) remove logging calls with \n und passe die logging aufrufe an (grade alle info) -# TODO 2. (pr) make cli flags to control verbosity (ich weiß grade nicht wie das gehen soll) -# TODO 3. (pr) be able to create a file config - path_to_logfile = 'spotpy.log' # Standard defintion of main logger From 0b318f0fec7b76a05edf1e3d4e3d0bb50b51c4a6 Mon Sep 17 00:00:00 2001 From: Chris Weber Date: Thu, 2 Apr 2020 14:25:18 +0200 Subject: [PATCH 03/18] Fix pep8 and unused comment --- src/spotpy/objectivefunctions.py | 3 +-- src/spotpy/spotpylogging.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/spotpy/objectivefunctions.py b/src/spotpy/objectivefunctions.py index 501d3391..94191459 100644 --- a/src/spotpy/objectivefunctions.py +++ b/src/spotpy/objectivefunctions.py @@ -11,9 +11,8 @@ """ from spotpy import spotpylogging import numpy as np -#logger.basicConfig(format='%(levelname)s: %(module)s.%(funcName)s(): %(message)s') -logger = spotpylogging.get_logger('objective_function') +logger = spotpylogging.get_logger('objective_functions') def bias(evaluation, simulation): """ diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index 23429eef..93f1c0d4 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ -Defines a standard logger for the main application, which every child can derive from. -Also it's possible to use the logger for the main application. +Defines a standard logger for the main application, which every child can + derive from. Also it's possible to use the logger for the main + application. """ import logging @@ -13,16 +14,17 @@ # Add logging to stdout handler_stdout = logging.StreamHandler() -formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') +formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + + '%(message)s') handler_stdout.setFormatter(formatter) spotpy_logger.addHandler(handler_stdout) # Add logging to file -handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments +handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments handler_file.setFormatter(formatter) spotpy_logger.addHandler(handler_file) -spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments +spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) @@ -30,4 +32,5 @@ def get_logger(name): """ Returns a new child logger for the main spotpy application logging Use this logger to return new childs of the main logger""" - return spotpy_logger.getChild(name) # Set the logger name, with the implementation class + # Set the logger name, with the implementation class + return spotpy_logger.getChild(name) From d241da235527f51e2fe2e989d39fc0d2b2b1acfb Mon Sep 17 00:00:00 2001 From: thouska Date: Thu, 16 Apr 2020 14:03:44 +0200 Subject: [PATCH 04/18] Fix doubled logging message https://stackoverflow.com/questions/6729268/log-messages-appearing-twice-with-python-logging --- src/spotpy/spotpylogging.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index 93f1c0d4..c5554f99 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -11,22 +11,25 @@ # Standard defintion of main logger spotpy_logger = logging.getLogger("spotpy") - -# Add logging to stdout -handler_stdout = logging.StreamHandler() -formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + - '%(message)s') -handler_stdout.setFormatter(formatter) -spotpy_logger.addHandler(handler_stdout) - -# Add logging to file -handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments -handler_file.setFormatter(formatter) -spotpy_logger.addHandler(handler_file) - -spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments -spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) - +if not spotpy_logger.handlers: + # create the handlers and call logger.addHandler(logging_handler) + # Add logging to stdout + handler_stdout = logging.StreamHandler() +# formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + +# '%(message)s') + formatter = logging.Formatter('%(message)s') + + handler_stdout.setFormatter(formatter) + spotpy_logger.addHandler(handler_stdout) + + # Add logging to file + handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments + handler_file.setFormatter(formatter) + spotpy_logger.addHandler(handler_file) + + spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments + spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) + def get_logger(name): """ Returns a new child logger for the main spotpy application logging From f5f667eb84df07c02796885e76492944e1062c3a Mon Sep 17 00:00:00 2001 From: thouska Date: Thu, 16 Apr 2020 14:11:09 +0200 Subject: [PATCH 05/18] Use different formats for logging in file and console --- src/spotpy/spotpylogging.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index c5554f99..e39e80f6 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -15,16 +15,16 @@ # create the handlers and call logger.addHandler(logging_handler) # Add logging to stdout handler_stdout = logging.StreamHandler() -# formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + -# '%(message)s') - formatter = logging.Formatter('%(message)s') + formatter_file = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + + '%(message)s') + formatter_stdout = logging.Formatter('%(message)s') - handler_stdout.setFormatter(formatter) + handler_stdout.setFormatter(formatter_stdout) spotpy_logger.addHandler(handler_stdout) # Add logging to file handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments - handler_file.setFormatter(formatter) + handler_file.setFormatter(formatter_file) spotpy_logger.addHandler(handler_file) spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments From 7fc07effe170733f0aea63ce839f3d588d699a57 Mon Sep 17 00:00:00 2001 From: thouska Date: Thu, 16 Apr 2020 15:25:18 +0200 Subject: [PATCH 06/18] Nuanced using of debug/info/error in algorithms --- src/spotpy/algorithms/abc.py | 8 ++--- src/spotpy/algorithms/dds.py | 4 +-- src/spotpy/algorithms/demcz.py | 14 ++++----- src/spotpy/algorithms/fast.py | 21 ++----------- src/spotpy/algorithms/fscabc.py | 10 +++---- src/spotpy/algorithms/lhs.py | 2 +- src/spotpy/algorithms/mcmc.py | 10 +++---- src/spotpy/algorithms/mle.py | 6 ++-- src/spotpy/algorithms/padds.py | 2 +- src/spotpy/algorithms/rope.py | 10 +++---- src/spotpy/algorithms/sa.py | 4 +++ src/spotpy/algorithms/sceua.py | 52 +++++++++++++++++---------------- src/spotpy/spotpylogging.py | 6 ++-- 13 files changed, 70 insertions(+), 79 deletions(-) diff --git a/src/spotpy/algorithms/abc.py b/src/spotpy/algorithms/abc.py index e96499eb..83a28ec5 100644 --- a/src/spotpy/algorithms/abc.py +++ b/src/spotpy/algorithms/abc.py @@ -104,7 +104,7 @@ def sample( work.append([like, randompar, like, randompar, c, p]) icall += 1 if self.status.stop: - self.logger.info('Stopping sampling') + self.logger.debug('Stopping sampling') break while icall < repetitions and gnrng > peps: @@ -139,7 +139,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.info('Stopping samplig') + self.logger.debug('Stopping samplig') break # Probability distribution for roulette wheel selection bn = [] for i, val in enumerate(work): @@ -189,7 +189,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.info('Stopping samplig') + self.logger.debug('Stopping samplig') break # Scout bee phase for i, val in enumerate(work): @@ -203,7 +203,7 @@ def sample( work[i][0] = clike icall += 1 if self.status.stop: - self.logger.info('Stopping samplig') + self.logger.debug('Stopping samplig') break gnrng = -self.status.objectivefunction_max if icall >= repetitions: diff --git a/src/spotpy/algorithms/dds.py b/src/spotpy/algorithms/dds.py index c803db50..29f878d3 100644 --- a/src/spotpy/algorithms/dds.py +++ b/src/spotpy/algorithms/dds.py @@ -332,7 +332,7 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.params_max = list(x_curr) self.params_max = self.fix_status_params_format(self.params_max) - self.logger.info('Best solution found has obj function value of %s at %s\n\n', \ + self.logger.debug('Best solution found has obj function value of %s at %s\n\n', \ objectivefunction_max, repitionno_best) debug_results.append( { @@ -363,7 +363,7 @@ def calc_initial_para_configuration( # by trying which randomized generated input matches best # initial_iterations is the number of function evaluations to initialize the DDS algorithm solution if initial_iterations > 1: - self.logger.info("Finding best starting point for trial %s using %s random samples.", \ + self.logger.debug("Finding best starting point for trial %s using %s random samples.", \ trial + 1, initial_iterations) repetions_left = ( repetitions - initial_iterations diff --git a/src/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py index d785df59..a4e0a7ad 100644 --- a/src/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -105,7 +105,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.info("ERROR Bounds have not the same lenghts as Parameterarray") + self.logger.error('ERROR Bounds have not the same lenghts as Parameterarray') return par def sample( @@ -222,9 +222,9 @@ def sample( # 3) and we have not done more than the maximum number of iterations while cur_iter < maxChainDraws: - self.logger.info("%s, %s", cur_iter, burnIn) + self.logger.debug("%s, %s", cur_iter, burnIn) if cur_iter == burnIn: - self.logger.info('starting') + self.logger.debug('starting') history.start_sampling() # every5th iteration allow a big jump @@ -536,8 +536,8 @@ def _random_no_replace(sampleSize, populationSize, numSamples): return samples -class _CovarianceConvergence: - +class _CovarianceConvergence(): + relativeVariances = {} def update(self, history, group): @@ -566,7 +566,7 @@ def rv(relevantHistory): projection = np.dot(np.linalg.inv(basis1), basis2) except np.linalg.linalg.LinAlgError as e: projection = np.array(basis1) * np.nan - spotpylogging.get_logger("_CovarianceConvergence()").logger.info("Exception happend!\nExcpetion:%s", e) + spotpylogging.get_logger("_CovarianceConvergence()").logger.debug("Exception happend!\nExcpetion:%s", e) # find the releative size in each of the basis1 directions return np.log(np.sum(projection**2, axis=0) ** 0.5) @@ -622,7 +622,7 @@ def _dream_proposals( return proposalVectors -class _GRConvergence: +class _GRConvergence(): """ Gelman Rubin convergence diagnostic calculator class. It currently only calculates the naive version found in the first paper. It does not check to see whether the variances have been diff --git a/src/spotpy/algorithms/fast.py b/src/spotpy/algorithms/fast.py index 5a404d21..c561f62e 100644 --- a/src/spotpy/algorithms/fast.py +++ b/src/spotpy/algorithms/fast.py @@ -149,7 +149,7 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): """ ) else: - self.logger.info( + self.logger.error( """ Error: Number of samples in model output file must be a multiple of D, where D is the number of parameters in your parameter file. @@ -169,24 +169,15 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): # Calculate and Output the First and Total Order Values if print_to_console: -<<<<<<< HEAD:src/spotpy/algorithms/fast.py - print("Parameter First Total") - Si = dict((k, [None] * D) for k in ["S1", "ST"]) -======= self.logger.info("Parameter First Total") Si = dict((k, [None] * D) for k in ['S1', 'ST']) ->>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py for i in range(D): l = np.arange(i * N, (i + 1) * N) Si["S1"][i] = self.compute_first_order(Y[l], N, M, omega[0]) Si["ST"][i] = self.compute_total_order(Y[l], N, omega[0]) if print_to_console: -<<<<<<< HEAD:src/spotpy/algorithms/fast.py - print("%s %f %f" % (parnames[i], Si["S1"][i], Si["ST"][i])) -======= self.logger.info("%s %f %f" % (parnames[i], Si['S1'][i], Si['ST'][i])) ->>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py return Si def compute_first_order(self, outputs, N, M, omega): @@ -213,15 +204,9 @@ def sample(self, repetitions, M=4): Maximum number of runs. """ self.set_repetiton(repetitions) -<<<<<<< HEAD:src/spotpy/algorithms/fast.py - print( - "Starting the FAST algotrithm with " + str(repetitions) + " repetitions..." - ) - print("Creating FAST Matrix") -======= self.logger.info('Starting the FAST algotrithm with '+str(repetitions)+ ' repetitions...') - self.logger.info('Creating FAST Matrix') ->>>>>>> a4673f4 (WIP: Adds logging):spotpy/algorithms/fast.py + + self.logger.debug('Creating FAST Matrix') # Get the names of the parameters to analyse names = self.parameter()["name"] # Get the minimum and maximum value for each parameter from the diff --git a/src/spotpy/algorithms/fscabc.py b/src/spotpy/algorithms/fscabc.py index 318b1a23..06005bb6 100644 --- a/src/spotpy/algorithms/fscabc.py +++ b/src/spotpy/algorithms/fscabc.py @@ -133,8 +133,8 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work.append([like, randompar, like, randompar, c, p]) icall += 1 if self.status.stop: - # icall = repetitions - self.logger.info("Stopping sampling") + #icall = repetitions + self.logger.debug('Stopping sampling') break # Bee Phases @@ -174,7 +174,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.info("Stopping sampling") + self.logger.debug('Stopping samplig') break # Fitness scaling @@ -223,7 +223,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.info("Stopping samplig") + self.logger.debug('Stopping samplig') break # Scout bee phase for i, val in enumerate(work): @@ -239,7 +239,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[i][0] = clike icall += 1 if self.status.stop: - self.logger.info("Stopping samplig") + self.logger.debug('Stopping samplig') break gnrng = -self.status.objectivefunction_max diff --git a/src/spotpy/algorithms/lhs.py b/src/spotpy/algorithms/lhs.py index 56ad2c24..72285b96 100644 --- a/src/spotpy/algorithms/lhs.py +++ b/src/spotpy/algorithms/lhs.py @@ -64,7 +64,7 @@ def sample(self, repetitions): self.logger.info( "Starting the LHS algotrithm with " + str(repetitions) + " repetitions..." ) - self.logger.info("Creating LatinHyperCube Matrix") + self.logger.debug("Creating LatinHyperCube Matrix") # Get the names of the parameters to analyse names = self.parameter()["name"] # Define the jump size between the parameter diff --git a/src/spotpy/algorithms/mcmc.py b/src/spotpy/algorithms/mcmc.py index 6c2481ba..e3e3ff5f 100644 --- a/src/spotpy/algorithms/mcmc.py +++ b/src/spotpy/algorithms/mcmc.py @@ -63,7 +63,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") + self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') return par def check_par_validity_reflect(self, par): @@ -81,7 +81,7 @@ def check_par_validity_reflect(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") + self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') return par def get_new_proposal_vector(self, old_par): @@ -117,7 +117,7 @@ def sample(self, repetitions, nChains=1): self.parameter()["minbound"], self.parameter()["maxbound"], ) - self.logger.info("Initialize ", self.nChains, " chain(s)...") + self.logger.debug("Initialize ", self.nChains, " chain(s)...") self.iter = 0 param_generator = ( (curChain, self.parameter()["random"]) @@ -132,7 +132,7 @@ def sample(self, repetitions, nChains=1): self.iter += 1 intervaltime = time.time() - self.logger.info("Beginn of Random Walk") + self.logger.debug('Beginn of Random Walk') # Walk through chains while self.iter <= repetitions - self.burnIn: param_generator = ( @@ -167,6 +167,6 @@ def sample(self, repetitions, nChains=1): ) * 100 ).strip("array([])") - self.logger.info(text) + self.logger.debug(text) intervaltime = time.time() self.final_call() diff --git a/src/spotpy/algorithms/mle.py b/src/spotpy/algorithms/mle.py index f481c825..d9e23fa3 100644 --- a/src/spotpy/algorithms/mle.py +++ b/src/spotpy/algorithms/mle.py @@ -59,7 +59,7 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.info("ERROR Bounds have not the same lenghts as Parameterarray") + self.logger.error('ERROR Bounds have not the same lenghts as Parameterarray') return par def sample(self, repetitions): @@ -79,7 +79,7 @@ def sample(self, repetitions): likes = [] pars = [] sims = [] - self.logger.info("burnIn...") + self.logger.debug('burnIn...') for i in range(burnIn): randompar = self.parameter()["random"] pars.append(randompar) @@ -90,7 +90,7 @@ def sample(self, repetitions): old_like = max(likes) old_par = pars[likes.index(old_like)] - self.logger.info("Beginn Random Walk") + self.logger.debug('Beginn Random Walk') for rep in range(repetitions - burnIn): # Suggest new candidate from Gaussian proposal distribution. # Use stepsize provided for every dimension. diff --git a/src/spotpy/algorithms/padds.py b/src/spotpy/algorithms/padds.py index ee32d3c2..860d3f4b 100644 --- a/src/spotpy/algorithms/padds.py +++ b/src/spotpy/algorithms/padds.py @@ -292,7 +292,7 @@ def sample( self.parameter_current, ) - self.logger.info( + self.logger.debug( "Best solution found has obj function value of " + str(self.best_value.best_obj_val) + " at " diff --git a/src/spotpy/algorithms/rope.py b/src/spotpy/algorithms/rope.py index 77b5bef8..3ac56515 100644 --- a/src/spotpy/algorithms/rope.py +++ b/src/spotpy/algorithms/rope.py @@ -167,7 +167,7 @@ def sample( if acttime - intervaltime >= 2: text = '1 Subset: Run %i of %i (best like=%g)' % ( rep, first_run, self.status.objectivefunction_max) - self.logger.info(text) + self.logger.debug(text) intervaltime = time.time() for subset in range(subsets - 1): @@ -204,7 +204,7 @@ def sample( likes.append(like) pars.append(ropepar) if self.status.stop: - self.logger.info('Stopping sampling') + self.logger.debug('Stopping sampling') break # Progress bar @@ -218,7 +218,7 @@ def sample( repetitions_following_runs, self.status.objectivefunction_max, ) - self.logger.info(text) + self.logger.debug(text) intervaltime = time.time() if self.status.stop: break @@ -230,10 +230,10 @@ def programm_depth(self, pars, runs): N, NP = X.shape text = str(N) + " input vectors with " + str(NP) + " parameters" - self.logger.info(text) + self.logger.debug(text) Ngen = int(runs) # int(N*(1/self.percentage)) - self.logger.info(("Generating " + str(Ngen) + " parameters:")) + self.logger.debug(("Generating " + str(Ngen) + " parameters:")) NPOSI = Ngen # Number of points to generate diff --git a/src/spotpy/algorithms/sa.py b/src/spotpy/algorithms/sa.py index fcf0499e..f85ed9df 100644 --- a/src/spotpy/algorithms/sa.py +++ b/src/spotpy/algorithms/sa.py @@ -62,7 +62,11 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: +<<<<<<< HEAD:src/spotpy/algorithms/sa.py self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") +======= + self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') +>>>>>>> f7fa4fd (Nuanced using of debug/info/error in algorithms):spotpy/algorithms/sa.py return par def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): diff --git a/src/spotpy/algorithms/sceua.py b/src/spotpy/algorithms/sceua.py index f268e89c..6880ea54 100644 --- a/src/spotpy/algorithms/sceua.py +++ b/src/spotpy/algorithms/sceua.py @@ -174,6 +174,8 @@ def sample( Number of loops executed at max in this function call """ self.set_repetiton(repetitions) + self.logger.info('Starting the SCEUA algotrithm with '+str(repetitions)+ ' repetitions...') + # Initialize SCE parameters: self.ngs = ngs randompar = self.parameter()["random"] @@ -206,7 +208,7 @@ def sample( icall = 0 xf = np.zeros(npt) - self.logger.info('Starting burn-in sampling...') + self.logger.debug('Starting burn-in sampling...') # Burn in param_generator = ((rep, x[rep]) for rep in range(int(npt))) @@ -216,7 +218,7 @@ def sample( xf[rep] = like icall += 1 if self.status.stop: - self.logger.info('Stopping sampling. Maximum number of repetitions reached already during burn-in') + self.logger.debug('Stopping samplig. Maximum number of repetitions reached already during burn-in') proceed = False break # Sort the population in order of increasing function values; @@ -256,15 +258,15 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - self.logger.info('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - self.logger.info('ON THE MAXIMUM NUMBER OF TRIALS ') - self.logger.info(repetitions) - self.logger.info('HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:') - self.logger.info(self.status.rep) - self.logger.info('OF THE INITIAL LOOP!') + self.logger.debug('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') + self.logger.debug('ON THE MAXIMUM NUMBER OF TRIALS ') + self.logger.debug(repetitions) + self.logger.debug('HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:') + self.logger.debug(self.status.rep) + self.logger.debug('OF THE INITIAL LOOP!') if gnrng < peps: - self.logger.info( + self.logger.debug( 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') # Begin evolution loops: @@ -294,7 +296,7 @@ def sample( and proceed == True ): nloop += 1 - self.logger.info('ComplexEvo loop #%d in progress...' % nloop) + self.logger.debug('ComplexEvo loop #%d in progress...' % nloop) # Loop on complexes (sub-populations); cx = np.zeros((self.npg, self.nopt)) cf = np.zeros((self.npg)) @@ -333,7 +335,7 @@ def sample( i, pars[i], sims[i], chains=i + 1, save_run=False ) self.discarded_runs += 1 - self.logger.info("Skipping saving") + self.logger.debug("Skipping saving") if ( self.breakpoint == "write" @@ -375,20 +377,20 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - self.logger.info("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") - self.logger.info("ON THE MAXIMUM NUMBER OF TRIALS ") - self.logger.info(repetitions) - self.logger.info("HAS BEEN EXCEEDED.") + self.logger.debug("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.debug("ON THE MAXIMUM NUMBER OF TRIALS ") + self.logger.debug(repetitions) + self.logger.debug("HAS BEEN EXCEEDED.") elif gnrng < peps: - self.logger.info( + self.logger.debug( "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" ) elif ( nloop >= kstop ): # necessary so that the area of high posterior density is visited as much as possible - self.logger.info( + self.logger.debug( "Objective function convergence criteria is now being updated and assessed..." ) absolute_change = ( @@ -399,13 +401,13 @@ def sample( criter_change_pcent = 0.0 else: criter_change_pcent = absolute_change / denominator - self.logger.info("Updated convergence criteria: %f" % criter_change_pcent) + self.logger.debug("Updated convergence criteria: %f" % criter_change_pcent) if criter_change_pcent <= pcento: - self.logger.info( + self.logger.debug( "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY LESS THAN THE USER-SPECIFIED THRESHOLD %f" % (kstop, pcento) ) - self.logger.info( + self.logger.debug( "CONVERGENCY HAS ACHIEVED BASED ON OBJECTIVE FUNCTION CRITERIA!!!" ) elif self.status.stop: @@ -415,14 +417,14 @@ def sample( # stop, if max number of loop iteration was reached elif max_loop_inc and nloop >= max_loop_inc: proceed = False - self.logger.info("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") + self.logger.debug("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") break # End of the Outer Loops - self.logger.info("SEARCH WAS STOPPED AT TRIAL NUMBER: %d" % self.status.rep) - self.logger.info("NUMBER OF DISCARDED TRIALS: %d" % self.discarded_runs) - self.logger.info("NORMALIZED GEOMETRIC RANGE = %f" % gnrng) - self.logger.info( + self.logger.debug("SEARCH WAS STOPPED AT TRIAL NUMBER: %d" % self.status.rep) + self.logger.debug("NUMBER OF DISCARDED TRIALS: %d" % self.discarded_runs) + self.logger.debug("NORMALIZED GEOMETRIC RANGE = %f" % gnrng) + self.logger.debug( "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY %f PERCENT" % (kstop, criter_change_pcent) ) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index e39e80f6..b345f0a1 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -26,8 +26,8 @@ handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments handler_file.setFormatter(formatter_file) spotpy_logger.addHandler(handler_file) - - spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments + + spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) @@ -36,4 +36,4 @@ def get_logger(name): Use this logger to return new childs of the main logger""" # Set the logger name, with the implementation class - return spotpy_logger.getChild(name) + return spotpy_logger.getChild(name) \ No newline at end of file From 47a51a045b8135756e5d4c19070f5676673b5c35 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 26 May 2021 18:27:38 +0200 Subject: [PATCH 07/18] Adds cli param (WIP) and logfile timestamp --- src/spotpy/spotpylogging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index b345f0a1..bc9d9f77 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -6,8 +6,9 @@ """ import logging +from datetime import datetime -path_to_logfile = 'spotpy.log' +path_to_logfile = '{}-spotpy.log'.format(datetime.isoformat(datetime.now())) # Standard defintion of main logger spotpy_logger = logging.getLogger("spotpy") From e3b260545bd5f72a3b3f72cd74575db040c2a812 Mon Sep 17 00:00:00 2001 From: Chris Date: Fri, 28 May 2021 11:12:21 +0200 Subject: [PATCH 08/18] Adds logdir option and click parameter delegation for _algorithm * Also new logging instantiation --- src/spotpy/algorithms/_algorithm.py | 10 +++- src/spotpy/spotpylogging.py | 71 ++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index 847c99f8..6fc23fd5 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -223,6 +223,7 @@ class _algorithm(object): _unaccepted_parameter_types = (parameter.List,) +<<<<<<< HEAD:src/spotpy/algorithms/_algorithm.py def __init__( self, spot_setup, @@ -241,9 +242,16 @@ def __init__( optimization_direction="grid", algorithm_name="", ): +======= + def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, + dbappend=False, parallel='seq', save_sim=True, breakpoint=None, + backup_every_rep=100, save_threshold=-np.inf, db_precision=np.float16, + sim_timeout=None, random_state=None, optimization_direction='grid', algorithm_name='', + quiet=False, logfile=None, logdir=None): +>>>>>>> d175060 (Adds logdir option and click parameter delegation for _algorithm):spotpy/algorithms/_algorithm.py # Instatiate logging - self.logger = spotpylogging.get_logger(self.__class__.__name__) + self.logger = spotpylogging.instantiate_logger(self.__class__.__name__, quiet, logfile, logdir) # Initialize the user defined setup class self.setup = spot_setup diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index bc9d9f77..31ffeff7 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -7,34 +7,63 @@ import logging from datetime import datetime +from os import path -path_to_logfile = '{}-spotpy.log'.format(datetime.isoformat(datetime.now())) +path_to_logfile = None # Standard defintion of main logger spotpy_logger = logging.getLogger("spotpy") -if not spotpy_logger.handlers: - # create the handlers and call logger.addHandler(logging_handler) - # Add logging to stdout - handler_stdout = logging.StreamHandler() - formatter_file = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s ' + - '%(message)s') - formatter_stdout = logging.Formatter('%(message)s') - - handler_stdout.setFormatter(formatter_stdout) - spotpy_logger.addHandler(handler_stdout) - - # Add logging to file - handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments - handler_file.setFormatter(formatter_file) - spotpy_logger.addHandler(handler_file) - spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments - spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) - +handler_stdout = None +handler_file = None + +formatter_file = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') +formatter_stdout = logging.Formatter('%(message)s') + +def instantiate_logger(name, quiet=None, logfile=None, logdir=None): + + + path_to_logdir = '.' + path_to_logfile = '{}-spotpy.log'.format(datetime.isoformat(datetime.now())) + + print(quiet, logfile, logdir) + + if not spotpy_logger.handlers: + # create the handlers and call logger.addHandler(logging_handler) + # Add logging to stdout + handler_stdout = logging.StreamHandler() + + handler_stdout.setFormatter(formatter_stdout) + spotpy_logger.addHandler(handler_stdout) + + if quiet: + handler_stdout.setLevel(logging.ERROR) + + # Add logging to file + if logdir is not None: + path_to_logdir = logdir + if logfile is not None: + path_to_logfile = logfile + path_to_logfile = path_to_logdir + path.sep + path_to_logfile + + handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments + handler_file.setFormatter(formatter_file) + spotpy_logger.addHandler(handler_file) + + spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments + spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) + + return get_logger(name) + #else: + #if quiet and not (logfile is None): + # raise RuntimeError("Logger is already instantiated!") + + # Set the logger name, with the implementation class def get_logger(name): """ Returns a new child logger for the main spotpy application logging Use this logger to return new childs of the main logger""" - # Set the logger name, with the implementation class - return spotpy_logger.getChild(name) \ No newline at end of file + + + return spotpy_logger.getChild(name) \ No newline at end of file From cda34893f4e342f03ef992a9bd237ab455275ff3 Mon Sep 17 00:00:00 2001 From: Chris Date: Tue, 1 Jun 2021 11:14:22 +0200 Subject: [PATCH 09/18] Fix tests and instantiation --- src/spotpy/spotpylogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index 31ffeff7..2def50be 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -53,7 +53,7 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) - return get_logger(name) + return get_logger(name) #else: #if quiet and not (logfile is None): # raise RuntimeError("Logger is already instantiated!") From 2ec590cbd91e780d49cd9e1d07ccbefb3d89f36c Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 6 Jun 2021 12:50:01 +0200 Subject: [PATCH 10/18] Various * Remove comments * Replaces last print and log messages with backspace char --- src/spotpy/algorithms/_algorithm.py | 6 ++++-- src/spotpy/database/__init__.py | 5 ++++- src/spotpy/spotpylogging.py | 25 ++++++++++--------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index 6fc23fd5..30178942 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -137,7 +137,8 @@ def print_status(self): self.last_print = time.time() def print_status_final(self): - self.logger.info('\n*** Final SPOTPY summary ***') + self.logger.info('') + self.logger.info('*** Final SPOTPY summary ***') self.logger.info('Total Duration: %s seconds' % str(round((time.time() - self.starttime), 2))) self.logger.info('Total Repetitions: %s', self.rep) @@ -168,7 +169,8 @@ def print_status_final(self): text = '%s: %g' % (self.parnames[i], self.params_max[i]) self.logger.info(text) - self.logger.info('******************************\n') + self.logger.info('******************************') + self.logger.info('') def __repr__(self): diff --git a/src/spotpy/database/__init__.py b/src/spotpy/database/__init__.py index 89ed286f..3263f87d 100644 --- a/src/spotpy/database/__init__.py +++ b/src/spotpy/database/__init__.py @@ -1,5 +1,8 @@ from importlib import import_module +from spotpy.spotpylogging import get_logger + +logger = get_logger('database.__init__') def __dir__(): """ @@ -19,7 +22,7 @@ def __dir__(): def __getattr__(name): names = __dir__() - print(names) + logger.info(names) if name in names: try: db_module = import_module("." + name, __name__) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index 2def50be..04f3b0d4 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -9,11 +9,10 @@ from datetime import datetime from os import path -path_to_logfile = None - # Standard defintion of main logger spotpy_logger = logging.getLogger("spotpy") +path_to_logfile = None handler_stdout = None handler_file = None @@ -21,13 +20,14 @@ formatter_stdout = logging.Formatter('%(message)s') def instantiate_logger(name, quiet=None, logfile=None, logdir=None): - + """ + Instantiate the logger, + if already instantiated returns a new child of the main logger + """ path_to_logdir = '.' path_to_logfile = '{}-spotpy.log'.format(datetime.isoformat(datetime.now())) - print(quiet, logfile, logdir) - if not spotpy_logger.handlers: # create the handlers and call logger.addHandler(logging_handler) # Add logging to stdout @@ -46,24 +46,19 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): path_to_logfile = logfile path_to_logfile = path_to_logdir + path.sep + path_to_logfile - handler_file = logging.FileHandler(path_to_logfile) # TODO this should be bound to cli arguments + handler_file = logging.FileHandler(path_to_logfile) handler_file.setFormatter(formatter_file) spotpy_logger.addHandler(handler_file) - spotpy_logger.setLevel(logging.INFO) # TODO this should be bound to cli arguments + spotpy_logger.setLevel(logging.INFO) spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) return get_logger(name) - #else: - #if quiet and not (logfile is None): - # raise RuntimeError("Logger is already instantiated!") - - # Set the logger name, with the implementation class + def get_logger(name): - """ Returns a new child logger for the main spotpy application logging + """ Returns a new child logger for the main spotpy application logging. Use this logger to return new childs of the main logger""" - - return spotpy_logger.getChild(name) \ No newline at end of file + return spotpy_logger.getChild(name) From 6001c7fb9abb223cdcda615e7e61e6a4b0678232 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 6 Jun 2021 13:30:09 +0200 Subject: [PATCH 11/18] Adds missing dependecy click to setup py --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index dddebd96..aeb79cee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ classifiers = [ dependencies = [ "numpy>=1.14.5", "scipy>=1.5.0", + "click", ] [project.optional-dependencies] From 315f1d3026ddc2d570827c7afacef1944ec1b567 Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 12 Jul 2021 13:24:48 +0200 Subject: [PATCH 12/18] Use os agnostic path objects --- src/spotpy/spotpylogging.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index 04f3b0d4..e4f8daa1 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -7,7 +7,7 @@ import logging from datetime import datetime -from os import path +from pathlib import PurePath # Standard defintion of main logger spotpy_logger = logging.getLogger("spotpy") @@ -25,8 +25,8 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): if already instantiated returns a new child of the main logger """ - path_to_logdir = '.' - path_to_logfile = '{}-spotpy.log'.format(datetime.isoformat(datetime.now())) + path_to_logdir = PurePath() # current directory is assumed + path_to_logfile = PurePath('{}-spotpy.log'.format(datetime.isoformat(datetime.now()))) if not spotpy_logger.handlers: # create the handlers and call logger.addHandler(logging_handler) @@ -44,7 +44,7 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): path_to_logdir = logdir if logfile is not None: path_to_logfile = logfile - path_to_logfile = path_to_logdir + path.sep + path_to_logfile + path_to_logfile = PurePath(path_to_logdir, path_to_logfile) handler_file = logging.FileHandler(path_to_logfile) handler_file.setFormatter(formatter_file) From 19715dd182b368e1155b16ddc38d5d965aa7e028 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 15:29:15 +0200 Subject: [PATCH 13/18] Add cli params after rebase --- src/spotpy/cli.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/spotpy/cli.py b/src/spotpy/cli.py index 9c0e4385..f61d06c4 100644 --- a/src/spotpy/cli.py +++ b/src/spotpy/cli.py @@ -84,6 +84,13 @@ def cli(): is_flag=True, help="Print only the configuration, can be used to create a config file with your_model.py > spotpy.conf", ) +@click.option('--quiet', '-q', is_flag=True, + help='Suppress any logging messages to stdout during computation') +@click.option('--logfile', type=click.Path(dir_okay=False), + help='Logging messages will be written to this file. If already exists, ' + 'contents are appended. A default filename is created with a current timestamp') +@click.option('--logdir', type=click.Path(file_okay=False), + help='Logging files will be written to this directory. Default is "."') def run(ctx, **kwargs): """ Runs a sampler for automatic calibration From 50860f5b01aef995b445c1131594cc870701b754 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 16:24:44 +0200 Subject: [PATCH 14/18] some wrongly merged code --- src/spotpy/algorithms/_algorithm.py | 11 +++-------- src/spotpy/algorithms/demcz.py | 2 +- src/spotpy/spotpylogging.py | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index 30178942..6a83c4f5 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -225,7 +225,6 @@ class _algorithm(object): _unaccepted_parameter_types = (parameter.List,) -<<<<<<< HEAD:src/spotpy/algorithms/_algorithm.py def __init__( self, spot_setup, @@ -243,14 +242,10 @@ def __init__( random_state=None, optimization_direction="grid", algorithm_name="", + quiet=False, + logfile=None, + logdir=None, ): -======= - def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, - dbappend=False, parallel='seq', save_sim=True, breakpoint=None, - backup_every_rep=100, save_threshold=-np.inf, db_precision=np.float16, - sim_timeout=None, random_state=None, optimization_direction='grid', algorithm_name='', - quiet=False, logfile=None, logdir=None): ->>>>>>> d175060 (Adds logdir option and click parameter delegation for _algorithm):spotpy/algorithms/_algorithm.py # Instatiate logging self.logger = spotpylogging.instantiate_logger(self.__class__.__name__, quiet, logfile, logdir) diff --git a/src/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py index a4e0a7ad..3a83d5a0 100644 --- a/src/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -622,7 +622,7 @@ def _dream_proposals( return proposalVectors -class _GRConvergence(): +class _GRConvergence: """ Gelman Rubin convergence diagnostic calculator class. It currently only calculates the naive version found in the first paper. It does not check to see whether the variances have been diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index e4f8daa1..bd657738 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -25,7 +25,7 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): if already instantiated returns a new child of the main logger """ - path_to_logdir = PurePath() # current directory is assumed + path_to_logdir = PurePath() # current directory is assumed path_to_logfile = PurePath('{}-spotpy.log'.format(datetime.isoformat(datetime.now()))) if not spotpy_logger.handlers: From 2340dce5817cf5e0f766697eb2c4a9683efefa20 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 18:07:19 +0200 Subject: [PATCH 15/18] apply black formats --- src/spotpy/algorithms/_algorithm.py | 87 +++++++++++++++------------ src/spotpy/algorithms/abc.py | 21 ++++--- src/spotpy/algorithms/dds.py | 18 ++++-- src/spotpy/algorithms/demcz.py | 22 ++++--- src/spotpy/algorithms/fast.py | 11 ++-- src/spotpy/algorithms/fscabc.py | 10 +-- src/spotpy/algorithms/list_sampler.py | 4 +- src/spotpy/algorithms/mc.py | 4 +- src/spotpy/algorithms/mcmc.py | 10 ++- src/spotpy/algorithms/mle.py | 8 ++- src/spotpy/algorithms/rope.py | 17 ++++-- src/spotpy/algorithms/sa.py | 19 +++--- src/spotpy/algorithms/sceua.py | 35 +++++++---- src/spotpy/cli.py | 24 +++++--- src/spotpy/database/__init__.py | 3 +- src/spotpy/database/csv.py | 4 +- src/spotpy/objectivefunctions.py | 71 ++++++---------------- src/spotpy/spotpylogging.py | 63 ++++++++++--------- 18 files changed, 235 insertions(+), 196 deletions(-) diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index 6a83c4f5..c32fd49b 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -36,18 +36,20 @@ class _RunStatistic(object): """ def __init__(self, repetitions, algorithm_name, optimization_direction, parnames): - self.optimization_direction = optimization_direction #grid, mazimize, minimize + self.optimization_direction = optimization_direction # grid, mazimize, minimize - self.logger = spotpylogging.get_logger("RunStatistic(%s)" % algorithm_name ) + self.logger = spotpylogging.get_logger("RunStatistic(%s)" % algorithm_name) - self.logger.info('Initializing the %s with %s repetitions', algorithm_name, repetitions) - if optimization_direction == 'minimize': + self.logger.info( + "Initializing the %s with %s repetitions", algorithm_name, repetitions + ) + if optimization_direction == "minimize": self.compare = self.minimizer - self.logger.info('The objective function will be minimized') - if optimization_direction == 'maximize': + self.logger.info("The objective function will be minimized") + if optimization_direction == "maximize": self.compare = self.maximizer - self.logger.info('The objective function will be maxiimized') - if optimization_direction == 'grid': + self.logger.info("The objective function will be maxiimized") + if optimization_direction == "grid": self.compare = self.grid self.rep = 0 @@ -137,41 +139,50 @@ def print_status(self): self.last_print = time.time() def print_status_final(self): - self.logger.info('') - self.logger.info('*** Final SPOTPY summary ***') - self.logger.info('Total Duration: %s seconds' % str(round((time.time() - self.starttime), 2))) - self.logger.info('Total Repetitions: %s', self.rep) - - if self.optimization_direction == 'minimize': - self.logger.info('Minimal objective value: %g' % (self.objectivefunction_min)) - self.logger.info('Corresponding parameter setting:') + self.logger.info("") + self.logger.info("*** Final SPOTPY summary ***") + self.logger.info( + "Total Duration: %s seconds" % str(round((time.time() - self.starttime), 2)) + ) + self.logger.info("Total Repetitions: %s", self.rep) + + if self.optimization_direction == "minimize": + self.logger.info( + "Minimal objective value: %g" % (self.objectivefunction_min) + ) + self.logger.info("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_min[i]) + text = "%s: %g" % (self.parnames[i], self.params_min[i]) self.logger.info(text) - if self.optimization_direction == 'maximize': - self.logger.info('Maximal objective value: %g' % (self.objectivefunction_max)) - self.logger.info('Corresponding parameter setting:') + if self.optimization_direction == "maximize": + self.logger.info( + "Maximal objective value: %g" % (self.objectivefunction_max) + ) + self.logger.info("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_max[i]) + text = "%s: %g" % (self.parnames[i], self.params_max[i]) self.logger.info(text) - if self.optimization_direction == 'grid': - self.logger.info('Minimal objective value: %g' % (self.objectivefunction_min)) - self.logger.info('Corresponding parameter setting:') + if self.optimization_direction == "grid": + self.logger.info( + "Minimal objective value: %g" % (self.objectivefunction_min) + ) + self.logger.info("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_min[i]) + text = "%s: %g" % (self.parnames[i], self.params_min[i]) self.logger.info(text) - self.logger.info('Maximal objective value: %g' % (self.objectivefunction_max)) - self.logger.info('Corresponding parameter setting:') + self.logger.info( + "Maximal objective value: %g" % (self.objectivefunction_max) + ) + self.logger.info("Corresponding parameter setting:") for i in range(self.parameters): - text = '%s: %g' % (self.parnames[i], self.params_max[i]) + text = "%s: %g" % (self.parnames[i], self.params_max[i]) self.logger.info(text) - self.logger.info('******************************') - self.logger.info('') - + self.logger.info("******************************") + self.logger.info("") def __repr__(self): return "Min objectivefunction: %g \n Max objectivefunction: %g" % ( @@ -242,13 +253,15 @@ def __init__( random_state=None, optimization_direction="grid", algorithm_name="", - quiet=False, - logfile=None, + quiet=False, + logfile=None, logdir=None, ): # Instatiate logging - self.logger = spotpylogging.instantiate_logger(self.__class__.__name__, quiet, logfile, logdir) + self.logger = spotpylogging.instantiate_logger( + self.__class__.__name__, quiet, logfile, logdir + ) # Initialize the user defined setup class self.setup = spot_setup @@ -299,11 +312,11 @@ def __init__( self._return_all_likes = False # allows multi-objective calibration if set to True, is set by the algorithm if breakpoint == "read" or breakpoint == "readandwrite": - self.logger.info('Reading backupfile') + self.logger.info("Reading backupfile") try: open(self.dbname + ".break") except FileNotFoundError: - self.logger.info('Backupfile not found') + self.logger.info("Backupfile not found") self.dbappend = True # Now a repeater (ForEach-object) is loaded @@ -377,7 +390,7 @@ def final_call(self): def _init_database(self, like, randompar, simulations): if self.dbinit: - self.logger.info('Initialize database...') + self.logger.info("Initialize database...") self.datawriter = database.get_datawriter( self.dbformat, diff --git a/src/spotpy/algorithms/abc.py b/src/spotpy/algorithms/abc.py index 83a28ec5..4003c5b6 100644 --- a/src/spotpy/algorithms/abc.py +++ b/src/spotpy/algorithms/abc.py @@ -78,7 +78,9 @@ def sample( sets the limit """ self.set_repetiton(repetitions) - self.logger.info('Starting the ABC algotrithm with %s repetitions...', repetitions) + self.logger.info( + "Starting the ABC algotrithm with %s repetitions...", repetitions + ) # Initialize ABC parameters: randompar = self.parameter()["random"] self.nopt = randompar.size @@ -104,7 +106,7 @@ def sample( work.append([like, randompar, like, randompar, c, p]) icall += 1 if self.status.stop: - self.logger.debug('Stopping sampling') + self.logger.debug("Stopping sampling") break while icall < repetitions and gnrng > peps: @@ -139,7 +141,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break # Probability distribution for roulette wheel selection bn = [] for i, val in enumerate(work): @@ -189,7 +191,7 @@ def sample( work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break # Scout bee phase for i, val in enumerate(work): @@ -203,17 +205,18 @@ def sample( work[i][0] = clike icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break gnrng = -self.status.objectivefunction_max if icall >= repetitions: - self.logger.info('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - self.logger.info('ON THE MAXIMUM NUMBER OF TRIALS ') + self.logger.info("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.info("ON THE MAXIMUM NUMBER OF TRIALS ") self.logger.info(repetitions) - self.logger.info('HAS BEEN EXCEEDED.') + self.logger.info("HAS BEEN EXCEEDED.") if gnrng < peps: self.logger.info( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN') + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE AT RUN" + ) self.logger.info(icall) self.final_call() diff --git a/src/spotpy/algorithms/dds.py b/src/spotpy/algorithms/dds.py index 29f878d3..e67cfcce 100644 --- a/src/spotpy/algorithms/dds.py +++ b/src/spotpy/algorithms/dds.py @@ -281,7 +281,9 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.parameter()["minbound"], self.parameter()["maxbound"], ) - self.logger.info('Starting the DDS algotrithm with %s repetitions...', repetitions) + self.logger.info( + "Starting the DDS algotrithm with %s repetitions...", repetitions + ) number_of_parameters = ( self.status.parameters @@ -332,8 +334,11 @@ def sample(self, repetitions, trials=1, x_initial=np.array([])): self.params_max = list(x_curr) self.params_max = self.fix_status_params_format(self.params_max) - self.logger.debug('Best solution found has obj function value of %s at %s\n\n', \ - objectivefunction_max, repitionno_best) + self.logger.debug( + "Best solution found has obj function value of %s at %s\n\n", + objectivefunction_max, + repitionno_best, + ) debug_results.append( { "sbest": self.params_max, @@ -363,8 +368,11 @@ def calc_initial_para_configuration( # by trying which randomized generated input matches best # initial_iterations is the number of function evaluations to initialize the DDS algorithm solution if initial_iterations > 1: - self.logger.debug("Finding best starting point for trial %s using %s random samples.", \ - trial + 1, initial_iterations) + self.logger.debug( + "Finding best starting point for trial %s using %s random samples.", + trial + 1, + initial_iterations, + ) repetions_left = ( repetitions - initial_iterations ) # use this to reduce number of fevals in DDS loop diff --git a/src/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py index 3a83d5a0..4c3255d4 100644 --- a/src/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -105,7 +105,9 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.error('ERROR Bounds have not the same lenghts as Parameterarray') + self.logger.error( + "ERROR Bounds have not the same lenghts as Parameterarray" + ) return par def sample( @@ -145,8 +147,7 @@ def sample( self.set_repetiton(repetitions) self.logger.info( - "Starting the DEMCz algotrithm with %s repetitions...", - repetitions + "Starting the DEMCz algotrithm with %s repetitions...", repetitions ) self.min_bound, self.max_bound = ( @@ -224,7 +225,7 @@ def sample( while cur_iter < maxChainDraws: self.logger.debug("%s, %s", cur_iter, burnIn) if cur_iter == burnIn: - self.logger.debug('starting') + self.logger.debug("starting") history.start_sampling() # every5th iteration allow a big jump @@ -364,12 +365,11 @@ def sample( self.iter = cur_iter self.burnIn = burnIn self.R = grConvergence.R - - self.logger.info('Gelman Rubin R=%s', self.R) + + self.logger.info("Gelman Rubin R=%s", self.R) self.status.rep = self.status.repetitions self.final_call() - def _update_accepts_ratio(self, weighting, acceptances): self.accepts_ratio = ( weighting * np.mean(acceptances) + (1 - weighting) * self.accepts_ratio @@ -536,8 +536,8 @@ def _random_no_replace(sampleSize, populationSize, numSamples): return samples -class _CovarianceConvergence(): - +class _CovarianceConvergence: + relativeVariances = {} def update(self, history, group): @@ -566,7 +566,9 @@ def rv(relevantHistory): projection = np.dot(np.linalg.inv(basis1), basis2) except np.linalg.linalg.LinAlgError as e: projection = np.array(basis1) * np.nan - spotpylogging.get_logger("_CovarianceConvergence()").logger.debug("Exception happend!\nExcpetion:%s", e) + spotpylogging.get_logger("_CovarianceConvergence()").logger.debug( + "Exception happend!\nExcpetion:%s", e + ) # find the releative size in each of the basis1 directions return np.log(np.sum(projection**2, axis=0) ** 0.5) diff --git a/src/spotpy/algorithms/fast.py b/src/spotpy/algorithms/fast.py index c561f62e..685ab465 100644 --- a/src/spotpy/algorithms/fast.py +++ b/src/spotpy/algorithms/fast.py @@ -170,14 +170,13 @@ def analyze(self, problem, Y, D, parnames, M=4, print_to_console=False): # Calculate and Output the First and Total Order Values if print_to_console: self.logger.info("Parameter First Total") - Si = dict((k, [None] * D) for k in ['S1', 'ST']) + Si = dict((k, [None] * D) for k in ["S1", "ST"]) for i in range(D): l = np.arange(i * N, (i + 1) * N) Si["S1"][i] = self.compute_first_order(Y[l], N, M, omega[0]) Si["ST"][i] = self.compute_total_order(Y[l], N, omega[0]) if print_to_console: - self.logger.info("%s %f %f" % - (parnames[i], Si['S1'][i], Si['ST'][i])) + self.logger.info("%s %f %f" % (parnames[i], Si["S1"][i], Si["ST"][i])) return Si def compute_first_order(self, outputs, N, M, omega): @@ -204,9 +203,11 @@ def sample(self, repetitions, M=4): Maximum number of runs. """ self.set_repetiton(repetitions) - self.logger.info('Starting the FAST algotrithm with '+str(repetitions)+ ' repetitions...') + self.logger.info( + "Starting the FAST algotrithm with " + str(repetitions) + " repetitions..." + ) - self.logger.debug('Creating FAST Matrix') + self.logger.debug("Creating FAST Matrix") # Get the names of the parameters to analyse names = self.parameter()["name"] # Get the minimum and maximum value for each parameter from the diff --git a/src/spotpy/algorithms/fscabc.py b/src/spotpy/algorithms/fscabc.py index 06005bb6..1c92e1bf 100644 --- a/src/spotpy/algorithms/fscabc.py +++ b/src/spotpy/algorithms/fscabc.py @@ -133,8 +133,8 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work.append([like, randompar, like, randompar, c, p]) icall += 1 if self.status.stop: - #icall = repetitions - self.logger.debug('Stopping sampling') + # icall = repetitions + self.logger.debug("Stopping sampling") break # Bee Phases @@ -174,7 +174,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break # Fitness scaling @@ -223,7 +223,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[rep][4] = work[rep][4] + 1 icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break # Scout bee phase for i, val in enumerate(work): @@ -239,7 +239,7 @@ def sample(self, repetitions, eb=48, a=(1 / 10), peps=0.0001, kpow=4, limit=None work[i][0] = clike icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig') + self.logger.debug("Stopping samplig") break gnrng = -self.status.objectivefunction_max diff --git a/src/spotpy/algorithms/list_sampler.py b/src/spotpy/algorithms/list_sampler.py index 45ebbf02..0da3bfb0 100644 --- a/src/spotpy/algorithms/list_sampler.py +++ b/src/spotpy/algorithms/list_sampler.py @@ -68,7 +68,9 @@ def sample(self, repetitions=None): self.set_repetiton(repetitions) # Initialization - self.logger.info("Starting the List sampler with " + str(repetitions) + " repetitions...") + self.logger.info( + "Starting the List sampler with " + str(repetitions) + " repetitions..." + ) param_generator = ( (rep, list(parameters[rep])) for rep in range(int(repetitions)) ) diff --git a/src/spotpy/algorithms/mc.py b/src/spotpy/algorithms/mc.py index ddd71440..edc4547c 100644 --- a/src/spotpy/algorithms/mc.py +++ b/src/spotpy/algorithms/mc.py @@ -61,7 +61,9 @@ def sample(self, repetitions): Maximum number of runs. """ self.set_repetiton(repetitions) - self.logger.info("Starting the MC algorithm with %s repetitions...", repetitions) + self.logger.info( + "Starting the MC algorithm with %s repetitions...", repetitions + ) # A generator that produces parametersets if called param_generator = ( (rep, self.parameter()["random"]) for rep in range(int(repetitions)) diff --git a/src/spotpy/algorithms/mcmc.py b/src/spotpy/algorithms/mcmc.py index e3e3ff5f..d60440f2 100644 --- a/src/spotpy/algorithms/mcmc.py +++ b/src/spotpy/algorithms/mcmc.py @@ -63,7 +63,9 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') + self.logger.error( + "ERROR: Bounds have not the same lenghts as Parameterarray" + ) return par def check_par_validity_reflect(self, par): @@ -81,7 +83,9 @@ def check_par_validity_reflect(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') + self.logger.error( + "ERROR: Bounds have not the same lenghts as Parameterarray" + ) return par def get_new_proposal_vector(self, old_par): @@ -132,7 +136,7 @@ def sample(self, repetitions, nChains=1): self.iter += 1 intervaltime = time.time() - self.logger.debug('Beginn of Random Walk') + self.logger.debug("Beginn of Random Walk") # Walk through chains while self.iter <= repetitions - self.burnIn: param_generator = ( diff --git a/src/spotpy/algorithms/mle.py b/src/spotpy/algorithms/mle.py index d9e23fa3..3e6da1b7 100644 --- a/src/spotpy/algorithms/mle.py +++ b/src/spotpy/algorithms/mle.py @@ -59,7 +59,9 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: - self.logger.error('ERROR Bounds have not the same lenghts as Parameterarray') + self.logger.error( + "ERROR Bounds have not the same lenghts as Parameterarray" + ) return par def sample(self, repetitions): @@ -79,7 +81,7 @@ def sample(self, repetitions): likes = [] pars = [] sims = [] - self.logger.debug('burnIn...') + self.logger.debug("burnIn...") for i in range(burnIn): randompar = self.parameter()["random"] pars.append(randompar) @@ -90,7 +92,7 @@ def sample(self, repetitions): old_like = max(likes) old_par = pars[likes.index(old_like)] - self.logger.debug('Beginn Random Walk') + self.logger.debug("Beginn Random Walk") for rep in range(repetitions - burnIn): # Suggest new candidate from Gaussian proposal distribution. # Use stepsize provided for every dimension. diff --git a/src/spotpy/algorithms/rope.py b/src/spotpy/algorithms/rope.py index 3ac56515..12a4f2d0 100644 --- a/src/spotpy/algorithms/rope.py +++ b/src/spotpy/algorithms/rope.py @@ -122,9 +122,11 @@ def sample( repetitions_following_runs = int((repetitions - first_run) / (subsets - 1)) # Needed to avoid an error in integer division somewhere in depth function if repetitions_following_runs % 2 != 0: - self.logger.info('Warning: Burn-in samples and total number of repetions are not compatible.\n' - 'SPOTPY will automatically adjust the number of total repetitions.') - repetitions_following_runs+=1 + self.logger.info( + "Warning: Burn-in samples and total number of repetions are not compatible.\n" + "SPOTPY will automatically adjust the number of total repetitions." + ) + repetitions_following_runs += 1 if NDIR is None: NDIR = int(repetitions_following_runs / 100.0) @@ -165,8 +167,11 @@ def sample( if self.status.stop: break if acttime - intervaltime >= 2: - text = '1 Subset: Run %i of %i (best like=%g)' % ( - rep, first_run, self.status.objectivefunction_max) + text = "1 Subset: Run %i of %i (best like=%g)" % ( + rep, + first_run, + self.status.objectivefunction_max, + ) self.logger.debug(text) intervaltime = time.time() @@ -204,7 +209,7 @@ def sample( likes.append(like) pars.append(ropepar) if self.status.stop: - self.logger.debug('Stopping sampling') + self.logger.debug("Stopping sampling") break # Progress bar diff --git a/src/spotpy/algorithms/sa.py b/src/spotpy/algorithms/sa.py index f85ed9df..c4a0315c 100644 --- a/src/spotpy/algorithms/sa.py +++ b/src/spotpy/algorithms/sa.py @@ -62,11 +62,9 @@ def check_par_validity(self, par): if par[i] > self.max_bound[i]: par[i] = self.max_bound[i] else: -<<<<<<< HEAD:src/spotpy/algorithms/sa.py - self.logger.info("ERROR: Bounds have not the same lenghts as Parameterarray") -======= - self.logger.error('ERROR: Bounds have not the same lenghts as Parameterarray') ->>>>>>> f7fa4fd (Nuanced using of debug/info/error in algorithms):spotpy/algorithms/sa.py + self.logger.error( + "ERROR: Bounds have not the same lenghts as Parameterarray" + ) return par def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): @@ -80,10 +78,13 @@ def sample(self, repetitions, Tini=80, Ntemp=50, alpha=0.99): """ self.set_repetiton(repetitions) self.logger.info( - 'Starting the SA algotrithm with '+str(repetitions)+ ' repetitions...' - ) - self.min_bound, self.max_bound = self.parameter()['minbound'], self.parameter()['maxbound'] - stepsizes = self.parameter()['step'] + "Starting the SA algotrithm with " + str(repetitions) + " repetitions..." + ) + self.min_bound, self.max_bound = ( + self.parameter()["minbound"], + self.parameter()["maxbound"], + ) + stepsizes = self.parameter()["step"] Eopt = 999999 Titer = Tini x = self.parameter()["optguess"] diff --git a/src/spotpy/algorithms/sceua.py b/src/spotpy/algorithms/sceua.py index 6880ea54..3bf906d4 100644 --- a/src/spotpy/algorithms/sceua.py +++ b/src/spotpy/algorithms/sceua.py @@ -174,7 +174,9 @@ def sample( Number of loops executed at max in this function call """ self.set_repetiton(repetitions) - self.logger.info('Starting the SCEUA algotrithm with '+str(repetitions)+ ' repetitions...') + self.logger.info( + "Starting the SCEUA algotrithm with " + str(repetitions) + " repetitions..." + ) # Initialize SCE parameters: self.ngs = ngs @@ -208,7 +210,7 @@ def sample( icall = 0 xf = np.zeros(npt) - self.logger.debug('Starting burn-in sampling...') + self.logger.debug("Starting burn-in sampling...") # Burn in param_generator = ((rep, x[rep]) for rep in range(int(npt))) @@ -218,7 +220,9 @@ def sample( xf[rep] = like icall += 1 if self.status.stop: - self.logger.debug('Stopping samplig. Maximum number of repetitions reached already during burn-in') + self.logger.debug( + "Stopping samplig. Maximum number of repetitions reached already during burn-in" + ) proceed = False break # Sort the population in order of increasing function values; @@ -258,16 +262,17 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - self.logger.debug('*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT') - self.logger.debug('ON THE MAXIMUM NUMBER OF TRIALS ') + self.logger.debug("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.debug("ON THE MAXIMUM NUMBER OF TRIALS ") self.logger.debug(repetitions) - self.logger.debug('HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:') + self.logger.debug("HAS BEEN EXCEEDED. SEARCH WAS STOPPED AT TRIAL NUMBER:") self.logger.debug(self.status.rep) - self.logger.debug('OF THE INITIAL LOOP!') + self.logger.debug("OF THE INITIAL LOOP!") if gnrng < peps: self.logger.debug( - 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') + "THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE" + ) # Begin evolution loops: nloop = 0 @@ -296,7 +301,7 @@ def sample( and proceed == True ): nloop += 1 - self.logger.debug('ComplexEvo loop #%d in progress...' % nloop) + self.logger.debug("ComplexEvo loop #%d in progress..." % nloop) # Loop on complexes (sub-populations); cx = np.zeros((self.npg, self.nopt)) cf = np.zeros((self.npg)) @@ -377,7 +382,9 @@ def sample( # Check for convergency; if self.status.rep >= repetitions: - self.logger.debug("*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT") + self.logger.debug( + "*** OPTIMIZATION SEARCH TERMINATED BECAUSE THE LIMIT" + ) self.logger.debug("ON THE MAXIMUM NUMBER OF TRIALS ") self.logger.debug(repetitions) self.logger.debug("HAS BEEN EXCEEDED.") @@ -401,7 +408,9 @@ def sample( criter_change_pcent = 0.0 else: criter_change_pcent = absolute_change / denominator - self.logger.debug("Updated convergence criteria: %f" % criter_change_pcent) + self.logger.debug( + "Updated convergence criteria: %f" % criter_change_pcent + ) if criter_change_pcent <= pcento: self.logger.debug( "THE BEST POINT HAS IMPROVED IN LAST %d LOOPS BY LESS THAN THE USER-SPECIFIED THRESHOLD %f" @@ -417,7 +426,9 @@ def sample( # stop, if max number of loop iteration was reached elif max_loop_inc and nloop >= max_loop_inc: proceed = False - self.logger.debug("THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED") + self.logger.debug( + "THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED" + ) break # End of the Outer Loops diff --git a/src/spotpy/cli.py b/src/spotpy/cli.py index f61d06c4..64470d6d 100644 --- a/src/spotpy/cli.py +++ b/src/spotpy/cli.py @@ -84,13 +84,23 @@ def cli(): is_flag=True, help="Print only the configuration, can be used to create a config file with your_model.py > spotpy.conf", ) -@click.option('--quiet', '-q', is_flag=True, - help='Suppress any logging messages to stdout during computation') -@click.option('--logfile', type=click.Path(dir_okay=False), - help='Logging messages will be written to this file. If already exists, ' - 'contents are appended. A default filename is created with a current timestamp') -@click.option('--logdir', type=click.Path(file_okay=False), - help='Logging files will be written to this directory. Default is "."') +@click.option( + "--quiet", + "-q", + is_flag=True, + help="Suppress any logging messages to stdout during computation", +) +@click.option( + "--logfile", + type=click.Path(dir_okay=False), + help="Logging messages will be written to this file. If already exists, " + "contents are appended. A default filename is created with a current timestamp", +) +@click.option( + "--logdir", + type=click.Path(file_okay=False), + help='Logging files will be written to this directory. Default is "."', +) def run(ctx, **kwargs): """ Runs a sampler for automatic calibration diff --git a/src/spotpy/database/__init__.py b/src/spotpy/database/__init__.py index 3263f87d..6e2f6aa0 100644 --- a/src/spotpy/database/__init__.py +++ b/src/spotpy/database/__init__.py @@ -2,7 +2,8 @@ from spotpy.spotpylogging import get_logger -logger = get_logger('database.__init__') +logger = get_logger("database.__init__") + def __dir__(): """ diff --git a/src/spotpy/database/csv.py b/src/spotpy/database/csv.py index a78bc54f..6ec414fa 100644 --- a/src/spotpy/database/csv.py +++ b/src/spotpy/database/csv.py @@ -25,7 +25,9 @@ def __init__(self, *args, **kwargs): # write header line self.db.write(str(",".join(self.header) + "\n")) else: - self.logger.info("* Appending to database file '{}.csv'.".format(self.dbname)) + self.logger.info( + "* Appending to database file '{}.csv'.".format(self.dbname) + ) # Continues writing file mode = "a" self.db = io.open(self.dbname + ".csv", mode) diff --git a/src/spotpy/objectivefunctions.py b/src/spotpy/objectivefunctions.py index 94191459..5f268d51 100644 --- a/src/spotpy/objectivefunctions.py +++ b/src/spotpy/objectivefunctions.py @@ -12,7 +12,8 @@ from spotpy import spotpylogging import numpy as np -logger = spotpylogging.get_logger('objective_functions') +logger = spotpylogging.get_logger("objective_functions") + def bias(evaluation, simulation): """ @@ -38,9 +39,7 @@ def bias(evaluation, simulation): return float(bias) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -67,9 +66,7 @@ def pbias(evaluation, simulation): return 100 * (float(np.nansum(sim - obs)) / float(np.nansum(obs))) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -102,9 +99,7 @@ def nashsutcliffe(evaluation, simulation): return 1 - (numerator / denominator) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -137,9 +132,7 @@ def lognashsutcliffe(evaluation, simulation, epsilon=0): / sum((np.log(e) - np.mean(np.log(e))) ** 2) ) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -164,9 +157,7 @@ def log_p(evaluation, simulation): normpdf = -(y**2) / 2 - np.log(np.sqrt(2 * np.pi)) return np.mean(normpdf) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -191,9 +182,7 @@ def correlationcoefficient(evaluation, simulation): correlation_coefficient = np.corrcoef(evaluation, simulation)[0, 1] return correlation_coefficient else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -217,9 +206,7 @@ def rsquared(evaluation, simulation): if len(evaluation) == len(simulation): return correlationcoefficient(evaluation, simulation) ** 2 else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -246,9 +233,7 @@ def mse(evaluation, simulation): mse = np.nanmean((obs - sim) ** 2) return mse else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -298,9 +283,7 @@ def mae(evaluation, simulation): mae = np.mean(np.abs(sim - obs)) return mae else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -326,9 +309,7 @@ def rrmse(evaluation, simulation): rrmse = rmse(evaluation, simulation) / np.mean(evaluation) return rrmse else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -363,9 +344,7 @@ def agreementindex(evaluation, simulation): ) return Agreement_index else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -392,9 +371,7 @@ def covariance(evaluation, simulation): covariance = np.mean((obs - obs_mean) * (sim - sim_mean)) return covariance else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -431,9 +408,7 @@ def decomposed_mse(evaluation, simulation): return decomposed_mse else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -461,9 +436,7 @@ def kge(evaluation, simulation, return_all=False): else: return kge else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -533,9 +506,7 @@ def kge_non_parametric(evaluation, simulation, return_all=False): else: return kge else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -553,9 +524,7 @@ def rsr(evaluation, simulation): rsr = rmse(evaluation, simulation) / np.std(evaluation) return rsr else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan @@ -586,9 +555,7 @@ def volume_error(evaluation, simulation): ve = np.sum(simulation - evaluation) / np.sum(evaluation) return float(ve) else: - logger.warning( - "evaluation and simulation lists does not have the same length." - ) + logger.warning("evaluation and simulation lists does not have the same length.") return np.nan diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index bd657738..e48031bc 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -16,8 +16,11 @@ handler_stdout = None handler_file = None -formatter_file = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s') -formatter_stdout = logging.Formatter('%(message)s') +formatter_file = logging.Formatter( + "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" +) +formatter_stdout = logging.Formatter("%(message)s") + def instantiate_logger(name, quiet=None, logfile=None, logdir=None): """ @@ -26,39 +29,41 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): """ path_to_logdir = PurePath() # current directory is assumed - path_to_logfile = PurePath('{}-spotpy.log'.format(datetime.isoformat(datetime.now()))) + path_to_logfile = PurePath( + "{}-spotpy.log".format(datetime.isoformat(datetime.now())) + ) if not spotpy_logger.handlers: - # create the handlers and call logger.addHandler(logging_handler) - # Add logging to stdout - handler_stdout = logging.StreamHandler() - - handler_stdout.setFormatter(formatter_stdout) - spotpy_logger.addHandler(handler_stdout) - - if quiet: - handler_stdout.setLevel(logging.ERROR) - - # Add logging to file - if logdir is not None: - path_to_logdir = logdir - if logfile is not None: - path_to_logfile = logfile - path_to_logfile = PurePath(path_to_logdir, path_to_logfile) - - handler_file = logging.FileHandler(path_to_logfile) - handler_file.setFormatter(formatter_file) - spotpy_logger.addHandler(handler_file) - - spotpy_logger.setLevel(logging.INFO) - spotpy_logger.info('Write logging output to file \'%s\'', path_to_logfile) + # create the handlers and call logger.addHandler(logging_handler) + # Add logging to stdout + handler_stdout = logging.StreamHandler() + + handler_stdout.setFormatter(formatter_stdout) + spotpy_logger.addHandler(handler_stdout) + + if quiet: + handler_stdout.setLevel(logging.ERROR) + + # Add logging to file + if logdir is not None: + path_to_logdir = logdir + if logfile is not None: + path_to_logfile = logfile + path_to_logfile = PurePath(path_to_logdir, path_to_logfile) + + handler_file = logging.FileHandler(path_to_logfile) + handler_file.setFormatter(formatter_file) + spotpy_logger.addHandler(handler_file) + + spotpy_logger.setLevel(logging.INFO) + spotpy_logger.info("Write logging output to file '%s'", path_to_logfile) return get_logger(name) - + def get_logger(name): - """ Returns a new child logger for the main spotpy application logging. + """Returns a new child logger for the main spotpy application logging. Use this logger to return new childs of the main logger""" - + return spotpy_logger.getChild(name) From 97add711fcc0336be423a469600ba1aaa214521e Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 18:09:27 +0200 Subject: [PATCH 16/18] apply isort formats --- src/spotpy/algorithms/_algorithm.py | 3 +-- src/spotpy/algorithms/demcz.py | 3 ++- src/spotpy/database/hdf5.py | 4 ++-- src/spotpy/objectivefunctions.py | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/spotpy/algorithms/_algorithm.py b/src/spotpy/algorithms/_algorithm.py index c32fd49b..bcb8c4ee 100644 --- a/src/spotpy/algorithms/_algorithm.py +++ b/src/spotpy/algorithms/_algorithm.py @@ -11,8 +11,7 @@ import numpy as np -from spotpy import database, parameter -from spotpy import spotpylogging +from spotpy import database, parameter, spotpylogging try: from queue import Queue diff --git a/src/spotpy/algorithms/demcz.py b/src/spotpy/algorithms/demcz.py index 4c3255d4..fe081afb 100644 --- a/src/spotpy/algorithms/demcz.py +++ b/src/spotpy/algorithms/demcz.py @@ -7,9 +7,10 @@ import numpy as np -from . import _algorithm from spotpy import spotpylogging +from . import _algorithm + class demcz(_algorithm): """ diff --git a/src/spotpy/database/hdf5.py b/src/spotpy/database/hdf5.py index 18cb54c8..e96fd248 100644 --- a/src/spotpy/database/hdf5.py +++ b/src/spotpy/database/hdf5.py @@ -12,10 +12,10 @@ import numpy as np -from .base import database - from spotpy import spotpylogging +from .base import database + try: import tables except ImportError: diff --git a/src/spotpy/objectivefunctions.py b/src/spotpy/objectivefunctions.py index 5f268d51..d2828ae4 100644 --- a/src/spotpy/objectivefunctions.py +++ b/src/spotpy/objectivefunctions.py @@ -9,9 +9,10 @@ This tool holds functions for statistic analysis. It takes Python-lists and returns the objective function value of interest. """ -from spotpy import spotpylogging import numpy as np +from spotpy import spotpylogging + logger = spotpylogging.get_logger("objective_functions") From 304eea720aaff4ab2b4d3bb36f217eb376e41e0b Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 18:18:36 +0200 Subject: [PATCH 17/18] apply pycodestyles --- src/spotpy/spotpylogging.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spotpy/spotpylogging.py b/src/spotpy/spotpylogging.py index e48031bc..588ab987 100644 --- a/src/spotpy/spotpylogging.py +++ b/src/spotpy/spotpylogging.py @@ -56,7 +56,8 @@ def instantiate_logger(name, quiet=None, logfile=None, logdir=None): spotpy_logger.addHandler(handler_file) spotpy_logger.setLevel(logging.INFO) - spotpy_logger.info("Write logging output to file '%s'", path_to_logfile) + spotpy_logger.info("Write logging output to file '%s'", + path_to_logfile) return get_logger(name) From b9022691e106f421fe3e3b1986a64db53a434b67 Mon Sep 17 00:00:00 2001 From: Chris Date: Sun, 18 Sep 2022 18:18:52 +0200 Subject: [PATCH 18/18] pin click version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aeb79cee..e2335644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ classifiers = [ dependencies = [ "numpy>=1.14.5", "scipy>=1.5.0", - "click", + "click==8.1.3", ] [project.optional-dependencies]