Merge pull request #330 from aphearin/hod_mock_memory_improvements

Mock factory memory improvements

.travis.yml

@@ -31,7 +31,7 @@ env:
         - SETUP_CMD='test'
         - PIP_DEPENDENCIES=''
         # Here we list the dependencies to be installed that are in conda
-        - CONDA_DEPENDENCIES='cython scipy beautifulsoup4 requests matplotlib h5py'
+        - CONDA_DEPENDENCIES='cython scipy beautifulsoup4 requests matplotlib!=1.5.1 h5py'
     matrix:
         # Make sure that egg_info works without dependencies
         - SETUP_CMD='egg_info'

halotools/empirical_models/factories/hod_mock_factory.py

@@ -79,14 +79,15 @@ def __init__(self, populate=True, **kwargs):
             Default is True. 
         """
 
-        super(HodMockFactory, self).__init__(populate=populate, **kwargs)
+        MockFactory.__init__(self, populate=populate, **kwargs)
 
-        self.preprocess_halo_catalog()
+        halocat = kwargs['halocat']
+        self.preprocess_halo_catalog(halocat)
 
         if populate is True:
             self.populate()
 
-    def preprocess_halo_catalog(self, apply_completeness_cut = True, **kwargs):
+    def preprocess_halo_catalog(self, halocat, apply_completeness_cut = True):
         """ Method to pre-process a halo catalog upon instantiation of 
         the mock object. This pre-processing includes identifying the 
         catalog columns that will be used by the model to create the mock, 
@@ -112,31 +113,33 @@ def preprocess_halo_catalog(self, apply_completeness_cut = True, **kwargs):
             `~halotools.empirical_models.model_defaults` will be used to populate the mock. 
             Default is True. 
         """
-
         ################ Make cuts on halo catalog ################
         # Select host halos only, since this is an HOD-style model
-        self.halo_table = SampleSelector.host_halo_selection(
-            table = self.halo_table)
+        halo_table = SampleSelector.host_halo_selection(table = halocat.halo_table)
 
         # make a conservative mvir completeness cut 
         # This cut can be controlled by changing sim_defaults.Num_ptcl_requirement
         if apply_completeness_cut is True:
-            cutoff_mvir = sim_defaults.Num_ptcl_requirement*self.halocat.particle_mass
-            mass_cut = (self.halo_table['halo_mvir'] > cutoff_mvir)
-            self.halo_table = self.halo_table[mass_cut]
+            cutoff_mvir = sim_defaults.Num_ptcl_requirement*self.particle_mass
+            mass_cut = (halo_table['halo_mvir'] > cutoff_mvir)
+            halo_table = halo_table[mass_cut]
 
         ############################################################
 
         ### Create new columns of the halo catalog, if applicable
         try:
             d = self.model.new_haloprop_func_dict
             for new_haloprop_key, new_haloprop_func in d.iteritems():
-                self.halo_table[new_haloprop_key] = new_haloprop_func(table = self.halo_table)
+                halo_table[new_haloprop_key] = new_haloprop_func(table = halo_table)
                 self.additional_haloprops.append(new_haloprop_key)
         except AttributeError:
             pass
 
-        self.model.build_lookup_tables(**kwargs)
+        self.halo_table = Table()
+        for key in self.additional_haloprops:
+            self.halo_table[key] = halo_table[key]
+
+        self.model.build_lookup_tables()
 
     def populate(self, **kwargs):
         """ Method populating halos with mock galaxies. 
@@ -178,11 +181,11 @@ def populate(self, **kwargs):
         # Positions are now assigned to all populations. 
         # Now enforce the periodic boundary conditions for all populations at once
         self.galaxy_table['x'] = model_helpers.enforce_periodicity_of_box(
-            self.galaxy_table['x'], self.halocat.Lbox)
+            self.galaxy_table['x'], self.Lbox)
         self.galaxy_table['y'] = model_helpers.enforce_periodicity_of_box(
-            self.galaxy_table['y'], self.halocat.Lbox)
+            self.galaxy_table['y'], self.Lbox)
         self.galaxy_table['z'] = model_helpers.enforce_periodicity_of_box(
-            self.galaxy_table['z'], self.halocat.Lbox)
+            self.galaxy_table['z'], self.Lbox)
 
         if hasattr(self.model, 'galaxy_selection_func'):
             mask = self.model.galaxy_selection_func(self.galaxy_table)

halotools/empirical_models/factories/mock_factory_template.py

@@ -64,18 +64,21 @@ def __init__(self, **kwargs):
 
         """
 
-        required_kwargs = ['halocat', 'model']
+        required_kwargs = ['model']
         model_helpers.bind_required_kwargs(required_kwargs, self, **kwargs)
 
         # Make any cuts on the halo catalog requested by the model
-        try: 
-            f = self.model.halo_selection_func
-            self.halo_table = f(self.halocat.halo_table)
-        except AttributeError:
-            self.halo_table = self.halocat.halo_table            
+        try:
+            halocat = kwargs['halocat']
+            self.model = kwargs['model']
+        except KeyError:
+            msg = ("\n``halocat`` and ``model`` are required ``MockFactory`` arguments\n")
+            raise HalotoolsError(msg)
+        for key in halocat.__dict__.keys():
+            setattr(self, key, halocat.__dict__[key])
 
         try:
-            self.ptcl_table = self.halocat.ptcl_table # pre-retrieve the particles from disk, if available
+            self.ptcl_table = halocat.ptcl_table # pre-retrieve the particles from disk, if available
         except:
             pass   
 
@@ -153,7 +156,7 @@ def number_density(self):
 
         """
         ngals = len(self.galaxy_table)
-        comoving_volume = self.halocat.Lbox**3
+        comoving_volume = self.Lbox**3
         return ngals/float(comoving_volume)
 
     def compute_galaxy_clustering(self, include_crosscorr = False, **kwargs):
@@ -271,7 +274,7 @@ def compute_galaxy_clustering(self, include_crosscorr = False, **kwargs):
             pos = three_dim_pos_bundle(table = self.galaxy_table, 
                 key1='x', key2='y', key3='z', mask=mask, return_complement=False)
             clustering = mock_observables.tpcf(
-                pos, rbins, period=self.halocat.Lbox, num_threads=num_threads, 
+                pos, rbins, period=self.Lbox, num_threads=num_threads, 
                 approx_cell1_size = [rmax, rmax, rmax])
             return rbin_centers, clustering
         else:
@@ -285,7 +288,7 @@ def compute_galaxy_clustering(self, include_crosscorr = False, **kwargs):
                 key1='x', key2='y', key3='z', mask=mask, return_complement=True)
             xi11, xi12, xi22 = mock_observables.tpcf(
                 sample1=pos, rbins=rbins, sample2=pos2, 
-                period=self.halocat.Lbox, num_threads=num_threads, 
+                period=self.Lbox, num_threads=num_threads, 
                 approx_cell1_size = [rmax, rmax, rmax])
             return rbin_centers, xi11, xi12, xi22 
 
@@ -382,7 +385,7 @@ def compute_galaxy_matter_cross_clustering(self, include_complement = False, **k
             raise HalotoolsError(msg)
 
         nptcl = np.max([model_defaults.default_nptcls, len(self.galaxy_table)])
-        ptcl_table = randomly_downsample_data(self.halocat.ptcl_table, nptcl)
+        ptcl_table = randomly_downsample_data(self.ptcl_table, nptcl)
         ptcl_pos = three_dim_pos_bundle(table = ptcl_table, 
             key1='x', key2='y', key3='z')
 
@@ -409,7 +412,7 @@ def compute_galaxy_matter_cross_clustering(self, include_complement = False, **k
                 key1='x', key2='y', key3='z', mask=mask, return_complement=False)
             clustering = mock_observables.tpcf(
                 sample1=pos, rbins=rbins, sample2=ptcl_pos, 
-                period=self.halocat.Lbox, num_threads=num_threads, do_auto=False, 
+                period=self.Lbox, num_threads=num_threads, do_auto=False, 
                 approx_cell1_size = [rmax, rmax, rmax])
             return rbin_centers, clustering
         else:
@@ -423,11 +426,11 @@ def compute_galaxy_matter_cross_clustering(self, include_complement = False, **k
                 key1='x', key2='y', key3='z', mask=mask, return_complement=True)
             clustering = mock_observables.tpcf(
                 sample1=pos, rbins=rbins, sample2=ptcl_pos, 
-                period=self.halocat.Lbox, num_threads=num_threads, do_auto=False, 
+                period=self.Lbox, num_threads=num_threads, do_auto=False, 
                 approx_cell1_size = [rmax, rmax, rmax])
             clustering2 = mock_observables.tpcf(
                 sample1=pos2, rbins=rbins, sample2=ptcl_pos, 
-                period=self.halocat.Lbox, num_threads=num_threads, do_auto=False, 
+                period=self.Lbox, num_threads=num_threads, do_auto=False, 
                 approx_cell1_size = [rmax, rmax, rmax])
             return rbin_centers, clustering, clustering2 
 
@@ -491,12 +494,12 @@ def compute_fof_group_ids(self, zspace = True,
         z = self.galaxy_table['z']
         if zspace is True:
             z += self.galaxy_table['vz']/100.
-            z = model_helpers.enforce_periodicity_of_box(z, self.halocat.Lbox)
+            z = model_helpers.enforce_periodicity_of_box(z, self.Lbox)
         pos = np.vstack((x, y, z)).T
 
         group_finder = mock_observables.FoFGroups(positions=pos, 
             b_perp = b_perp, b_para = b_para, 
-            Lbox = self.halocat.Lbox, num_threads = num_threads)
+            Lbox = self.Lbox, num_threads = num_threads)
 
         return group_finder.group_ids
 

halotools/empirical_models/factories/model_factory_template.py

@@ -125,26 +125,26 @@ def test_consistency_with_existing_mock(**kwargs):
                 redshift = kwargs['halocat'].redshift
             else:
                 redshift = sim_defaults.default_redshift
-            if abs(redshift - self.mock.halocat.redshift) > 0.05:
-                raise HalotoolsError(inconsistent_redshift_error_msg % (redshift, self.mock.halocat.redshift))
+            if abs(redshift - self.mock.redshift) > 0.05:
+                raise HalotoolsError(inconsistent_redshift_error_msg % (redshift, self.mock.redshift))
 
             if 'simname' in kwargs:
                 simname = kwargs['simname']
             elif 'halocat' in kwargs:
                 simname = kwargs['halocat'].simname
             else:
                 simname = sim_defaults.default_simname
-            if simname != self.mock.halocat.simname:
-                raise HalotoolsError(inconsistent_simname_error_msg % (self.mock.halocat.simname, simname))
+            if simname != self.mock.simname:
+                raise HalotoolsError(inconsistent_simname_error_msg % (self.mock.simname, simname))
 
             if 'halo_finder' in kwargs:
                 halo_finder = kwargs['halo_finder']
             elif 'halocat' in kwargs:
                 halo_finder = kwargs['halocat'].halo_finder
             else:
                 halo_finder = sim_defaults.default_halo_finder
-            if halo_finder != self.mock.halocat.halo_finder:
-                raise HalotoolsError(inconsistent_halo_finder_error_msg % (self.mock.halocat.halo_finder,halo_finder ))
+            if halo_finder != self.mock.halo_finder:
+                raise HalotoolsError(inconsistent_halo_finder_error_msg % (self.mock.halo_finder,halo_finder ))
 
 
         try:

halotools/empirical_models/factories/subhalo_mock_factory.py

@@ -8,6 +8,8 @@
 import numpy as np
 from copy import copy 
 
+from astropy.table import Table 
+
 from .mock_factory_template import MockFactory
 
 from .. import model_helpers, model_defaults
@@ -49,16 +51,17 @@ def __init__(self, populate=True, **kwargs):
             with mock galaxies and their observable properties. Default is ``True``. 
         """
 
-        super(SubhaloMockFactory, self).__init__(populate = populate, **kwargs)
+        MockFactory.__init__(self, populate = populate, **kwargs)
+        halocat = kwargs['halocat']
 
         # Pre-compute any additional halo properties required by the model
-        self.preprocess_halo_catalog()
+        self.preprocess_halo_catalog(halocat)
         self.precompute_galprops()
 
         if populate is True:
             self.populate()
 
-    def preprocess_halo_catalog(self):
+    def preprocess_halo_catalog(self, halocat):
         """ Method to pre-process a halo catalog upon instantiation of the mock object. 
 
         New columns are added to the ``halo_table`` according to any entries in the 
@@ -69,16 +72,20 @@ def preprocess_halo_catalog(self):
         :ref:`new_haloprop_func_dict_mechanism`
 
         """
-
+        halo_table = halocat.halo_table
         ### Create new columns of the halo catalog, if applicable
         try:
             d = self.model.new_haloprop_func_dict
             for new_haloprop_key, new_haloprop_func in d.iteritems():
-                self.halo_table[new_haloprop_key] = new_haloprop_func(table = self.halo_table)
+                halo_table[new_haloprop_key] = new_haloprop_func(table = halo_table)
                 self.additional_haloprops.append(new_haloprop_key)
         except AttributeError:
             pass
 
+        self.halo_table = Table()
+        for key in self.additional_haloprops:
+            self.halo_table[key] = halo_table[key]
+
 
     def precompute_galprops(self):
         """ Method pre-processes the input subhalo catalog, and pre-computes