From 5b0e18ee3304c273cc77f217253e7de07c056e66 Mon Sep 17 00:00:00 2001 From: philippkraft Date: Tue, 23 Oct 2018 10:27:59 +0200 Subject: [PATCH 1/3] Added a Pickable replacement parameter.ParameterSet for the namedtuple as algorithm.partype. --- spotpy/algorithms/_algorithm.py | 3 +- spotpy/parameter.py | 176 ++++++++++++++++++---- spotpy/unittests/test_setup_parameters.py | 76 +++++++++- 3 files changed, 221 insertions(+), 34 deletions(-) diff --git a/spotpy/algorithms/_algorithm.py b/spotpy/algorithms/_algorithm.py index e38412ac..5eb37189 100644 --- a/spotpy/algorithms/_algorithm.py +++ b/spotpy/algorithms/_algorithm.py @@ -164,8 +164,7 @@ def __init__(self, spot_setup, dbname=None, dbformat=None, dbinit=True, self.parnames = param_info['name'] # Create a type to hold the parameter values using a namedtuple - self.partype = parameter.get_namedtuple_from_paramnames( - self.setup, self.parnames) + self.partype = parameter.ParameterSet(param_info) # use alt_objfun if alt_objfun is defined in objectivefunctions, # else self.setup.objectivefunction diff --git a/spotpy/parameter.py b/spotpy/parameter.py index 1a096b8a..5a234290 100644 --- a/spotpy/parameter.py +++ b/spotpy/parameter.py @@ -12,8 +12,6 @@ if sys.version_info[0] >= 3: unicode = str - -from collections import namedtuple from itertools import cycle @@ -517,6 +515,148 @@ def __init__(self, *args, **kwargs): """ super(Triangular, self).__init__(rnd.triangular, 'Triangular', *args, **kwargs) + +class ParameterSet(object): + """ + A Pickable parameter set to use named parameters in a setup + Is not created by a user directly, but in algorithm. + Older versions used a namedtuple, which is not pickable. + + An instance of ParameterSet is sent to the users setup.simulate method. + + Usage: + >>> ps = ParameterSet(...) + + Update values by arguments or keyword arguments + + >>> ps(0, 1, 2) + >>> ps(a=1, c=2) + + Assess parameter values of this parameter set + + >>> ps[0] == ps['a'] == ps.a + + A parameter set is a sequence: + + >>> list(ps) + + Assess the parameter set properties as arrays + >>> [ps.maxbound, ps.minbound, ps.optguess, ps.step, ps.random] + + + """ + def __init__(self, param_info): + """ + Creates a set of parameters from a parameter info array. + To create the parameter set from a setup use either: + >>> setup = ... + >>> ps = ParameterSet(get_parameters_array(setup)) + + or you can just use a function for this: + + >>> ps = create_set(setup) + + :param param_info: A record array containing the properties of the parameters + of this set. + """ + self.__lookup = dict(("p" + x if x.isdigit() else x, i) for i, x in enumerate(param_info['name'])) + self.__info = param_info + + def __call__(self, *values, **kwargs): + """ + Populates the values ('random') of the parameter set with new data + :param values: Contains the new values or omitted. + If given, the number of values needs to match the number + of parameters + :param kwargs: Can be used to set only single parameter values + :return: + """ + if values: + if len(self.__info) != len(values): + raise ValueError('Given values do are not the same length as the parameter set') + self.__info['random'][:] = values + for k in kwargs: + try: + self.__info['random'][self.__lookup[k]] = kwargs[k] + except KeyError: + raise TypeError('{} is not a parameter of this set'.format(k)) + return self + + def __len__(self): + return len(self.__info['random']) + + def __iter__(self): + return iter(self.__info['random']) + + def __getitem__(self, item): + """ + Provides item access + >>> ps[0] == ps['a'] + + :raises: KeyError, IndexError and TypeError + """ + try: + return self.__info['random'][item] + except TypeError: + positem = self.__lookup[item] + return self.__info['random'][positem] + + def __setitem__(self, key, value): + """ + Provides setting of item + >>> ps[0] = 1 + >>> ps['a'] = 2 + """ + if key in self.__lookup: + key = self.__lookup[key] + self.__info['random'][key] = value + + def __getattr__(self, item): + """ + Provides the attribute access like + >>> print(ps.a) + """ + if item.startswith('_'): + raise AttributeError('{} is not a member of this parameter set'.format(item)) + elif item in self.__lookup: + return self.__info['random'][self.__lookup[item]] + elif item in self.__info.dtype.names: + return self.__info[item] + else: + raise AttributeError('{} is not a member of this parameter set'.format(item)) + + def __setattr__(self, key, value): + """ + Provides setting of attributes + >>> ps.a = 2 + """ + # Allow normal usage + if key.startswith('_') or key not in self.__lookup: + return object.__setattr__(self, key, value) + else: + self.__info['random'][self.__lookup[key]] = value + + def __str__(self): + return 'parameters({})'.format( + ', '.join('{}={:g}'.format(k, self.__info['random'][v]) + for k, v in self.__lookup.items() + ) + ) + + def __repr__(self): + return 'spotpy.parameter.ParameterSet()' + + def __dir__(self): + """ + Helps to show the field names in an interactive environment like IPython. + See: http://ipython.readthedocs.io/en/stable/config/integrating.html + + :return: List of method names and fields + """ + attrs = [attr for attr in vars(type(self)) if not attr.startswith('_')] + return attrs + list(self.__info['name']) + list(self.__info.dtype.names) + + def get_classes(): keys = [] current_module = sys.modules[__name__] @@ -525,6 +665,7 @@ def get_classes(): keys.append(key) return keys + def generate(parameters): """ This function generates a parameter set from a list of parameter objects. The parameter set @@ -561,7 +702,6 @@ def get_parameters_array(setup, unaccepted_parameter_types=()): # function param_arrays = [] # Get parameters defined with the setup class - #setup_parameters = checked_parameter_types(, unaccepted_parameter_types) setup_parameters = get_parameters_from_setup(setup) check_parameter_types(setup_parameters, unaccepted_parameter_types) param_arrays.append( @@ -587,8 +727,7 @@ def find_constant_parameters(parameter_array): return (parameter_array['maxbound'] - parameter_array['minbound'] == 0.0) - -def create_set(setup, valuetype='optguess', **kwargs): +def create_set(setup, valuetype='random', **kwargs): """ Returns a named tuple holding parameter values, to be used with the simulation method of a setup @@ -611,34 +750,13 @@ def create_set(setup, valuetype='optguess', **kwargs): params = get_parameters_array(setup) # Create the namedtuple from the parameter names - partype = get_namedtuple_from_paramnames(setup, params['name']) - - # Use the generated values from the distribution - pardict = dict(zip(params['name'], params[valuetype])) - - # Overwrite parameters with keyword arguments - pardict.update(kwargs) + partype = ParameterSet(params) # Return the namedtuple with fitting names - return partype(**pardict) - - -def get_namedtuple_from_paramnames(owner, parnames): - """ - Returns the namedtuple classname for parameter names - :param owner: Owner of the parameters, usually the spotpy setup - :param parnames: Sequence of parameter names - :return: Class - """ - - # Get name of owner class - typename = type(owner).__name__ - parnames = ["p" + x if x.isdigit() else x for x in list(parnames)] - return namedtuple('Par_' + typename, # Type name created from the setup name - parnames) # get parameter names + return partype(*params[valuetype], **kwargs) -def get_constant_indices(setup, unaccepted_parameter_types=(Constant)): +def get_constant_indices(setup, unaccepted_parameter_types=(Constant,)): """ Returns a list of the class defined parameters, and overwrites the names of the parameters. diff --git a/spotpy/unittests/test_setup_parameters.py b/spotpy/unittests/test_setup_parameters.py index 63aac486..e1aab7f5 100644 --- a/spotpy/unittests/test_setup_parameters.py +++ b/spotpy/unittests/test_setup_parameters.py @@ -83,6 +83,7 @@ class SpotSetupParameterList(SpotSetupBase): def __init__(self): self.parameters = [parameter.Uniform(name, -1, 1) for name in 'abcd'] + class SpotSetupMixedParameterList(SpotSetupBase): """ A Test case with two parameters as class parameters (a,b) @@ -90,9 +91,78 @@ class SpotSetupMixedParameterList(SpotSetupBase): """ a = parameter.Uniform(0, 1) b = parameter.Uniform(1, 2) + def parameters(self): return parameter.generate([parameter.Uniform(name, -1, 1) for name in 'cd']) - + + +class TestParameterSet(unittest.TestCase): + def setUp(self): + model = SpotSetupParameterFunction() + param_info = model.parameters() + self.ps = parameter.ParameterSet(param_info) + + def test_create(self): + self.assertEqual(type(self.ps), parameter.ParameterSet) + + def test_assign(self): + values = [1] * len(self.ps) + self.ps(values) + self.assertEquals(list(self.ps), values) + # Test if wrong number of parameters raises + with self.assertRaises(ValueError): + self.ps(values[:-1]) + + def test_iter(self): + values = [1] * len(self.ps) + self.ps(values) + ps_values = list(self.ps) + self.assertEquals(values, ps_values) + + def test_getitem(self): + values = [1] * len(self.ps) + self.ps(values) + self.assertEquals(self.ps['a'], 1.0) + self.assertEquals(self.ps[0], 1.0) + + def test_getattr(self): + values = [1] * len(self.ps) + self.ps(values) + + with self.assertRaises(AttributeError): + _ = self.ps.__x + + self.assertEquals(self.ps.a, 1.0) + self.assertEquals(list(self.ps.random), list(self.ps), 'Access to random variable does not equal list of names') + + with self.assertRaises(AttributeError): + _ = self.ps.x + + def test_setattr(self): + self.ps.a = 2 + self.assertEquals(self.ps[0], 2) + + def test_dir(self): + values = [1] * len(self.ps) + self.ps(values) + + attrs = dir(self.ps) + for param in self.ps.name: + self.assertIn(param, attrs, 'Attribute {} not found in {}'.format(param, self.ps)) + for prop in ['maxbound', 'minbound', 'name', 'optguess', 'random', 'step']: + self.assertIn(prop, attrs, 'Property {} not found in {}'.format(prop, self.ps)) + + def test_str(self): + values = [1] * len(self.ps) + self.ps(values) + self.assertEquals(str(self.ps), 'parameters(a=1, b=1, c=1, d=1)') + + def test_repr(self): + values = [1] * len(self.ps) + self.ps(values) + self.assertEquals(repr(self.ps), 'spotpy.parameter.ParameterSet()') + + class TestSetupVariants(unittest.TestCase): def setUp(self): # Get all Setups from this module @@ -102,8 +172,8 @@ def test_exists(self): self.assertGreater(len(self.objects), 0) def parameter_count_test(self, o): - params = parameter.create_set(o) - param_names = ','.join(pn for pn in params._fields) + params = parameter.create_set(o, valuetype='optguess') + param_names = ','.join(pn for pn in params.name) self.assertEqual(len(params), 4, '{} should have 4 parameters, but found only {} ({})' .format(o, len(params), param_names)) self.assertEqual(param_names, 'a,b,c,d', '{} Parameter names should be "a,b,c,d" but got "{}"' From 027425b48d4b455f2b8d2131bb3d601e415dad3b Mon Sep 17 00:00:00 2001 From: philippkraft Date: Tue, 23 Oct 2018 16:16:33 +0200 Subject: [PATCH 2/3] Fixed ParameterSet.__getitem__ and the tests to use *values. fixed in the GUI the use of _fields variable of the namedtuple --- spotpy/gui/mpl.py | 2 +- spotpy/parameter.py | 8 +++----- spotpy/unittests/test_setup_parameters.py | 16 ++++++++-------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/spotpy/gui/mpl.py b/spotpy/gui/mpl.py index 5732bd59..09e3179f 100644 --- a/spotpy/gui/mpl.py +++ b/spotpy/gui/mpl.py @@ -191,7 +191,7 @@ def run(self, _=None): sim = self.setup.simulation(parset) objf = as_scalar(self.setup.objectivefunction(sim, self.setup.evaluation())) label = ('{:0.4g}=M('.format(objf) - + ', '.join('{f}={v:0.4g}'.format(f=f, v=v) for f, v in zip(parset._fields, parset)) + + ', '.join('{f}={v:0.4g}'.format(f=f, v=v) for f, v in zip(parset.name, parset)) + ')') self.lines.extend(self.ax.plot(sim, '-', label=label)) self.ax.legend() diff --git a/spotpy/parameter.py b/spotpy/parameter.py index 5a234290..84385288 100644 --- a/spotpy/parameter.py +++ b/spotpy/parameter.py @@ -595,11 +595,9 @@ def __getitem__(self, item): :raises: KeyError, IndexError and TypeError """ - try: - return self.__info['random'][item] - except TypeError: - positem = self.__lookup[item] - return self.__info['random'][positem] + if type(item) is str: + item = self.__lookup[item] + return self.__info['random'][item] def __setitem__(self, key, value): """ diff --git a/spotpy/unittests/test_setup_parameters.py b/spotpy/unittests/test_setup_parameters.py index e1aab7f5..620794be 100644 --- a/spotpy/unittests/test_setup_parameters.py +++ b/spotpy/unittests/test_setup_parameters.py @@ -107,27 +107,27 @@ def test_create(self): def test_assign(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) self.assertEquals(list(self.ps), values) # Test if wrong number of parameters raises with self.assertRaises(ValueError): - self.ps(values[:-1]) + self.ps(*values[:-1]) def test_iter(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) ps_values = list(self.ps) self.assertEquals(values, ps_values) def test_getitem(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) self.assertEquals(self.ps['a'], 1.0) self.assertEquals(self.ps[0], 1.0) def test_getattr(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) with self.assertRaises(AttributeError): _ = self.ps.__x @@ -144,7 +144,7 @@ def test_setattr(self): def test_dir(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) attrs = dir(self.ps) for param in self.ps.name: @@ -154,12 +154,12 @@ def test_dir(self): def test_str(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) self.assertEquals(str(self.ps), 'parameters(a=1, b=1, c=1, d=1)') def test_repr(self): values = [1] * len(self.ps) - self.ps(values) + self.ps(*values) self.assertEquals(repr(self.ps), 'spotpy.parameter.ParameterSet()') From 0e0e423e8c4b4bee62f4d853b2b82fc4fff9c784 Mon Sep 17 00:00:00 2001 From: philippkraft Date: Thu, 25 Oct 2018 13:28:36 +0200 Subject: [PATCH 3/3] Changed ParameterSet.__str__ to be deterministic in Python <= 3.5 --- spotpy/parameter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spotpy/parameter.py b/spotpy/parameter.py index 84385288..a13ec483 100644 --- a/spotpy/parameter.py +++ b/spotpy/parameter.py @@ -636,8 +636,8 @@ def __setattr__(self, key, value): def __str__(self): return 'parameters({})'.format( - ', '.join('{}={:g}'.format(k, self.__info['random'][v]) - for k, v in self.__lookup.items() + ', '.join('{}={:g}'.format(k, self.__info['random'][i]) + for i, k in enumerate(self.__info['name']) ) )