From ed7c28dedf69e238efff186a94bcacc4292aed38 Mon Sep 17 00:00:00 2001 From: lu-kas <> Date: Mon, 22 Jun 2020 09:39:31 +0200 Subject: [PATCH 1/5] SCEUA: added maximal number of loops to be exectued during a sampling call --- spotpy/algorithms/sceua.py | 61 ++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/spotpy/algorithms/sceua.py b/spotpy/algorithms/sceua.py index bdcc5f9a..e212ab22 100644 --- a/spotpy/algorithms/sceua.py +++ b/spotpy/algorithms/sceua.py @@ -5,10 +5,10 @@ :author: Tobias Houska and Stijn Van Hoey ''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals + + + + from . import _algorithm import numpy as np @@ -63,6 +63,8 @@ def __init__(self, *args, **kwargs): kwargs['optimization_direction'] = 'minimize' kwargs['algorithm_name'] = 'Shuffled Complex Evolution (SCE-UA) algorithm' super(sceua, self).__init__(*args, **kwargs) + + print("LUKAS version anchor") def simulate(self, id_params_tuple): @@ -128,7 +130,7 @@ def simulate(self, id_params_tuple): # Replace the complex back into the population; return igs, likes, pars, sims, cx, cf, k1, k2, discarded_runs - def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.0000001): + def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.0000001, max_loop_inc=None): """ Samples from parameter distributions using SCE-UA (Duan, 2004), converted to python by Van Hoey (2011), restructured and parallelized by Houska et al (2015). @@ -146,6 +148,8 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 the percentage change allowed in the past kstop loops below which convergence is assumed to be achieved. peps: float Value of the normalized geometric range of the parameters in the population below which convergence is deemed achieved. + max_loop_inc: int + Number of loops executed at max in this function call """ self.set_repetiton(repetitions) # Initialize SCE parameters: @@ -163,6 +167,9 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 bound = self.bu - self.bl # np.array self.stochastic_parameters = bound != 0 proceed = True + + # burnin_only, needed to indictat if only the burnin phase should be run + burnin_only = False if self.breakpoint == 'read' or self.breakpoint == 'readandwrite': data_frombreak = self.read_breakdata(self.dbname) icall = data_frombreak[0] @@ -177,7 +184,7 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 icall = 0 xf = np.zeros(npt) - print ('Starting burn-in sampling...') + print('Starting burn-in sampling...') # Burn in param_generator = ((rep, x[rep]) for rep in range(int(npt))) @@ -194,6 +201,17 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 idx = np.argsort(xf) xf = np.sort(xf) x = x[idx, :] + + print('Burn-in sampling completed...') + + # write brakpoint if only a single generation shall be computed and + # the main loop will not be executed + if (self.breakpoint == 'write' or self.breakpoint == 'readandwrite') \ + and max_loop_inc == 1: + work = (self.status.rep, (x, xf), gnrng) + self.write_breakdata(self.dbname, work) + burnin_only = True + else: raise ValueError("Don't know the breakpoint keyword {}".format(self.breakpoint)) @@ -221,16 +239,23 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 print( 'THE POPULATION HAS CONVERGED TO A PRESPECIFIED SMALL PARAMETER SPACE') - print ('Burn-in sampling completed...') + + # Begin evolution loops: nloop = 0 criter = [] criter_change_pcent = 1e+5 - - self.repeat.setphase('ComplexEvo') - print ('Starting Complex Evolution...') proceed = True + + # if only burnin, stop the following while loop to be started + if burnin_only: + proceed = False + print('ONLY THE BURNIN PHASE WAS COMPUTED') + else: + self.repeat.setphase('ComplexEvo') + print('Starting Complex Evolution...') + while icall < repetitions and gnrng > peps and criter_change_pcent > pcento and proceed == True: nloop += 1 print ('ComplexEvo loop #%d in progress...' % nloop) @@ -260,10 +285,10 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 self.discarded_runs+=1 print('Skipping saving') - if self.breakpoint == 'write' or self.breakpoint == 'readandwrite'\ - and self.status.rep >= self.backup_every_rep: - work = (self.status.rep, (x, xf), gnrng) - self.write_breakdata(self.dbname, work) + if self.breakpoint == 'write' or self.breakpoint == 'readandwrite'\ + and self.status.rep >= self.backup_every_rep: + work = (self.status.rep, (x, xf), gnrng) + self.write_breakdata(self.dbname, work) # End of Loop on Complex Evolution; @@ -315,7 +340,13 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 'CONVERGENCY HAS ACHIEVED BASED ON OBJECTIVE FUNCTION CRITERIA!!!') elif self.status.stop: proceed = False - break + break + + # stop, if max number of loop iteration was reached + elif max_loop_inc and nloop >= max_loop_inc: + proceed = False + print('THE MAXIMAL NUMBER OF LOOPS PER EXECUTION WAS REACHED') + break # End of the Outer Loops print('SEARCH WAS STOPPED AT TRIAL NUMBER: %d' % self.status.rep) From e89fbb7c5f220dbaae04f7bfde403878ef96affd Mon Sep 17 00:00:00 2001 From: lu-kas <> Date: Mon, 22 Jun 2020 11:00:00 +0200 Subject: [PATCH 2/5] moved write breakpoint --- spotpy/algorithms/sceua.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/spotpy/algorithms/sceua.py b/spotpy/algorithms/sceua.py index e212ab22..601fc594 100644 --- a/spotpy/algorithms/sceua.py +++ b/spotpy/algorithms/sceua.py @@ -170,6 +170,7 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 # burnin_only, needed to indictat if only the burnin phase should be run burnin_only = False + if self.breakpoint == 'read' or self.breakpoint == 'readandwrite': data_frombreak = self.read_breakdata(self.dbname) icall = data_frombreak[0] @@ -202,16 +203,11 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 xf = np.sort(xf) x = x[idx, :] - print('Burn-in sampling completed...') - - # write brakpoint if only a single generation shall be computed and - # the main loop will not be executed - if (self.breakpoint == 'write' or self.breakpoint == 'readandwrite') \ - and max_loop_inc == 1: - work = (self.status.rep, (x, xf), gnrng) - self.write_breakdata(self.dbname, work) + if max_loop_inc == 1: burnin_only = True + print('Burn-in sampling completed...') + else: raise ValueError("Don't know the breakpoint keyword {}".format(self.breakpoint)) @@ -249,9 +245,15 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 proceed = True # if only burnin, stop the following while loop to be started + # write brakpoint if only a single generation shall be computed and + # the main loop will not be executed if burnin_only: + if self.breakpoint == 'write' or self.breakpoint == 'readandwrite': + work = (self.status.rep, (x, xf), gnrng) + self.write_breakdata(self.dbname, work) proceed = False print('ONLY THE BURNIN PHASE WAS COMPUTED') + else: self.repeat.setphase('ComplexEvo') print('Starting Complex Evolution...') From 9676bfc06d0929a5c5828c0ccac630d4a487936d Mon Sep 17 00:00:00 2001 From: lu-kas <> Date: Fri, 4 Sep 2020 10:39:54 +0200 Subject: [PATCH 3/5] cleanup for PR --- spotpy/algorithms/sceua.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/spotpy/algorithms/sceua.py b/spotpy/algorithms/sceua.py index 601fc594..7d901f6b 100644 --- a/spotpy/algorithms/sceua.py +++ b/spotpy/algorithms/sceua.py @@ -5,10 +5,10 @@ :author: Tobias Houska and Stijn Van Hoey ''' - - - - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals from . import _algorithm import numpy as np @@ -64,8 +64,6 @@ def __init__(self, *args, **kwargs): kwargs['algorithm_name'] = 'Shuffled Complex Evolution (SCE-UA) algorithm' super(sceua, self).__init__(*args, **kwargs) - print("LUKAS version anchor") - def simulate(self, id_params_tuple): """This overwrites the simple wrapper function of _algorithms.py @@ -253,7 +251,7 @@ def sample(self, repetitions, ngs=20, kstop=100, pcento=0.0000001, peps=0.000000 self.write_breakdata(self.dbname, work) proceed = False print('ONLY THE BURNIN PHASE WAS COMPUTED') - + else: self.repeat.setphase('ComplexEvo') print('Starting Complex Evolution...') @@ -446,4 +444,4 @@ def _sampleinputmatrix(self, nrows, npars): x = np.zeros((nrows, npars)) for i in range(nrows): x[i, :] = self.parameter()['random'] - return x \ No newline at end of file + return x From 4418bc215a60af2af4e09b12d40bec009797b4e4 Mon Sep 17 00:00:00 2001 From: lu-kas <> Date: Fri, 4 Sep 2020 10:41:48 +0200 Subject: [PATCH 4/5] removed whitespaces --- spotpy/algorithms/sceua.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spotpy/algorithms/sceua.py b/spotpy/algorithms/sceua.py index 7d901f6b..86679eb1 100644 --- a/spotpy/algorithms/sceua.py +++ b/spotpy/algorithms/sceua.py @@ -5,9 +5,9 @@ :author: Tobias Houska and Stijn Van Hoey ''' -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from __future__ import unicode_literals from . import _algorithm import numpy as np @@ -64,7 +64,6 @@ def __init__(self, *args, **kwargs): kwargs['algorithm_name'] = 'Shuffled Complex Evolution (SCE-UA) algorithm' super(sceua, self).__init__(*args, **kwargs) - def simulate(self, id_params_tuple): """This overwrites the simple wrapper function of _algorithms.py and makes a two phase mpi parallelization possbile: From bc7a24ad13ca43472790433d7a03525b0008dc3f Mon Sep 17 00:00:00 2001 From: thouska Date: Fri, 4 Sep 2020 14:09:01 +0200 Subject: [PATCH 5/5] Updated version number Slight performance improvement for sceua example --- setup.py | 2 +- spotpy/__init__.py | 2 +- spotpy/examples/tutorial_sceua_hymod.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 8b786c28..728f6be6 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name = 'spotpy', - version = '1.5.10', + version = '1.5.11', description = 'A Statistical Parameter Optimization Tool', long_description=open(os.path.join(os.path.dirname(__file__), "README.rst")).read(), diff --git a/spotpy/__init__.py b/spotpy/__init__.py index 89b1022b..322f1456 100644 --- a/spotpy/__init__.py +++ b/spotpy/__init__.py @@ -40,4 +40,4 @@ from . import describe # Contains some helper functions to describe samplers and set-ups from .hydrology import signatures # Quantifies goodness of fit between simulation and evaluation data with hydrological signatures -__version__ = '1.5.10' \ No newline at end of file +__version__ = '1.5.11' \ No newline at end of file diff --git a/spotpy/examples/tutorial_sceua_hymod.py b/spotpy/examples/tutorial_sceua_hymod.py index 25c7129c..3fd3626f 100644 --- a/spotpy/examples/tutorial_sceua_hymod.py +++ b/spotpy/examples/tutorial_sceua_hymod.py @@ -21,10 +21,10 @@ # Initialize the Hymod example # In this case, we tell the setup which algorithm we want to use, so # we can use this exmaple for different algorithms - spot_setup=spot_setup(users_objective_function=spotpy.objectivefunctions.rmse) + spot_setup=spot_setup(spotpy.objectivefunctions.rmse) #Select number of maximum allowed repetitions - rep=500 + rep=1000 filename = 'SCEUA_hymod' # Create the SCE-UA sampler of spotpy, alt_objfun is set to None to force SPOTPY # to jump into the def objectivefunction in the spot_setup class (default is @@ -32,7 +32,7 @@ sampler=spotpy.algorithms.sceua(spot_setup, dbname='SCEUA_hymod', dbformat='csv') #Start the sampler, one can specify ngs, kstop, peps and pcento id desired - sampler.sample(rep)#,ngs=10, kstop=50, peps=0.1, pcento=0.1) + sampler.sample(rep, ngs=7, kstop=3, peps=0.1, pcento=0.1) # Load the results gained with the sceua sampler, stored in SCEUA_hymod.csv results = spotpy.analyser.load_csv_results('SCEUA_hymod')