From 2f8306de5115fd084f00b56c551daf2b6289d434 Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Thu, 21 Mar 2024 16:50:18 +0100 Subject: [PATCH 1/3] Basic LephareInformer class started Working through the goldenspike notebook one step at a time --- src/rail/estimation/algos/lephare.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/rail/estimation/algos/lephare.py b/src/rail/estimation/algos/lephare.py index c2850cf..dc0d5f9 100644 --- a/src/rail/estimation/algos/lephare.py +++ b/src/rail/estimation/algos/lephare.py @@ -1,5 +1,21 @@ # import qp -from rail.estimation.estimator import CatEstimator +from rail.estimation.estimator import CatEstimator, CatInformer + + + +class LephareInformer(CatInformer): + """Inform stage for LephareEstimator + + This class will set templates and filters required for the photoz estimation + """ + name = "LephareInformer" + config_options = CatInformer.config_options.copy() + config_options.update() + + + def run(self): + """Compute the best fit prior parameters + """ class LephareEstimator(CatEstimator): """LePhare-base CatEstimator From efaf63a956bd7e801c1a9129e2acd5a97b95cdd2 Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Mon, 25 Mar 2024 21:42:09 +0100 Subject: [PATCH 2/3] LephareInformer class passing basic notebook tests We are overwriting config from the para file via python possibly unnecesarily Can we confirm config settings for LSST --- src/rail/estimation/algos/lephare.py | 142 ++++++++++++++++-- src/rail/estimation/algos/lsst.para | 210 +++++++++++++++++++++++++++ 2 files changed, 339 insertions(+), 13 deletions(-) create mode 100644 src/rail/estimation/algos/lsst.para diff --git a/src/rail/estimation/algos/lephare.py b/src/rail/estimation/algos/lephare.py index dc0d5f9..482c379 100644 --- a/src/rail/estimation/algos/lephare.py +++ b/src/rail/estimation/algos/lephare.py @@ -1,38 +1,154 @@ # import qp from rail.estimation.estimator import CatEstimator, CatInformer - +from rail.core.common_params import SHARED_PARAMS +import lephare as lp class LephareInformer(CatInformer): """Inform stage for LephareEstimator - - This class will set templates and filters required for the photoz estimation + + This class will set templates and filters required for photoz estimation """ + name = "LephareInformer" config_options = CatInformer.config_options.copy() - config_options.update() - - + config_options.update( + zmin=SHARED_PARAMS, + zmax=SHARED_PARAMS, + nzbins=SHARED_PARAMS, + nondetect_val=SHARED_PARAMS, + mag_limits=SHARED_PARAMS, + bands=SHARED_PARAMS, + err_bands=SHARED_PARAMS, + ref_band=SHARED_PARAMS, + redshift_col=SHARED_PARAMS, + ) + + def __init__(self, args, comm=None): + """Init function, init config stuff (COPIED from rail_bpz)""" + CatInformer.__init__(self, args, comm=comm) + # Default local parameters + self.config_file = "lsst.para" + print("config file: " + self.config_file) + self.lephare_config = lp.read_config(self.config_file) + + def _set_config(self, lephare_config): + """Update the lephare config + + Parameters + ========== + lepahre_config : `dict` + A dictionary of the lephare config keywords. + """ + self.lephare_config = lephare_config + + def _create_filter_library(self): + """Make the filter library files in lephare format""" + # load filters from config file + filterLib = lp.FilterSvc.from_config(self.config_file) + # Get location to store filter files + filter_output = os.path.join( + os.environ["LEPHAREWORK"], "filt", self.lephare_config["FILTER_FILE"].value + ) + # Write filter files + lp.write_output_filter( + filter_output + ".dat", filter_output + ".doc", filterLib + ) + + def _create_sed_library(self): + """Make the SED binary library files in lephare format. + + We separately create the star, quasar and galaxy libraries. + """ + sedlib = lp.Sedtolib(config_keymap=self.lephare_config) + sedlib.run(typ="STAR", star_sed="$LEPHAREDIR/examples/STAR_MOD_ALL.list") + sedlib.run(typ="QSO", qso_sed="$LEPHAREDIR/sed/QSO/SALVATO09/AGN_MOD.list") + sedlib.run(typ="GAL", gal_sed="$LEPHAREDIR/examples/COSMOS_MOD.list") + + def _create_mag_library(self): + """Make the magnitudes library file in lephare format. + + We separately create the star, quasar and galaxy libraries. + + TODO: replace hardcoded config options with class config options. + """ + maglib = lp.MagGal(config_keymap=self.lephare_config) + maglib.run(typ="STAR", lib_ascii="YES") + maglib.run( + typ="QSO", + lib_ascii="YES", + mod_extinc="0,1000", + eb_v="0.,0.1,0.2,0.3", + extinc_law="SB_calzetti.dat", + ) + maglib.run( + typ="GAL", + lib_ascii="YES", + mod_extinc="18,26,26,33,26,33,26,33", + extinc_law=( + "SMC_prevot.dat,SB_calzetti.dat," + + "SB_calzetti_bump1.dat,SB_calzetti_bump2.dat" + ), + em_lines="EMP_UV", + em_dispersion="0.5,0.75,1.,1.5,2.", + ) + def run(self): - """Compute the best fit prior parameters + """Run rail_lephare inform stage. + + This is the basic informer which takes the config and templates and + makes the inputs required for the run. + + In addition to the three lephare stages making the filter, sed, and + magnitude libraries we also do some tasks required by all rail inform + stages. """ + # Set training data required for all informers? + if self.config.hdf5_groupname: + training_data = self.get_data("input")[self.config.hdf5_groupname] + else: # pragma: no cover + training_data = self.get_data("input") + + # Get number of sources + ngal = len(training_data[self.config.ref_band]) + + # The three main lephare specific inform tasks + self._create_filter_library() + self._create_sed_library() + self._create_mag_library() + + # Spectroscopic redshifts + self.szs = training_data[self.config.redshift_col] + + # Give principle inform config 'model' to instance. + self.model = dict(config_file=self.config_file) + self.add_data("model", self.model) + class LephareEstimator(CatEstimator): - """LePhare-base CatEstimator - """ + """LePhare-base CatEstimator""" - name = 'LephareEstimator' + name = "LephareEstimator" config_options = CatEstimator.config_options.copy() # Add Lephare-specific configuration options here - config_options.update() + config_options.update( + zmin=SHARED_PARAMS, + zmax=SHARED_PARAMS, + nzbins=SHARED_PARAMS, + nondetect_val=SHARED_PARAMS, + mag_limits=SHARED_PARAMS, + bands=SHARED_PARAMS, + ref_band=SHARED_PARAMS, + err_bands=SHARED_PARAMS, + redshift_col=SHARED_PARAMS, + ) def __init__(self, args, comm=None): CatEstimator.__init__(self, args, comm=comm) def _process_chunk(self, start, end, data, first): - """Placeholder for the estimation calculation - """ + """Placeholder for the estimation calculation""" # some calculations take place here # pdfs = lephare.estimate(data) diff --git a/src/rail/estimation/algos/lsst.para b/src/rail/estimation/algos/lsst.para new file mode 100644 index 0000000..79d3052 --- /dev/null +++ b/src/rail/estimation/algos/lsst.para @@ -0,0 +1,210 @@ +############################################################################################## +########### CREATION OF LIBRARIES FROM SEDs List ############# +########### Run : $ZPHOTDIR/source/sedtolib -t (S/Q/G) -c zphot.para ############# +############################################################################################## +# +#------------------- STELLAR LIBRARY (ASCII SEDs) --------------------------- +STAR_SED STAR_SWIRE.list # STAR list (full path) +STAR_LIB LSST_STAR_BIN # Binary STAR LIBRARY (-> $ZPHOTWORK/lib_bin/*) +STAR_FSCALE 3.432E-09 # Arbitrary Flux Scale +# +#------------------- QSO LIBRARY (ASCII SEDs) --------------------------- +QSO_SED AGN_LONSDALE.list # QSO list (full path) +QSO_LIB LSST_QSO_BIN # Binary QSO LIBRARY (-> $ZPHOTWORK/lib_bin/*) +QSO_FSCALE 1. # Arbitrary Flux Scale +# +#------------------- GALAXY LIBRARY (ASCII or BINARY SEDs) --------------------------- +GAL_SED CE_MOD.list # GALAXMuzzin09_SEDY list (full path) +GAL_LIB LSST_GAL_BIN # Binary GAL LIBRARY (-> $ZPHOTWORK/lib_bin/*) +GAL_FSCALE 1. # Arbitrary Flux Scale +#SEL_AGE /data/zphot_vers25_03_03/sed/GAL/HYPERZ/AGE_GISSEL_HZ.dat # List of Age for GISSEL(full path) +AGE_RANGE 0.,15.e9 # Age Min-Max in yr + + +# +############################################################################################## +########### FILTERS ############# +########### Run : $ZPHOTDIR/source/filter -c zphot.para ############# +############################################################################################## + +# +FILTER_REP $LEPHAREDIR/filt # Repository in which the filters are stored +FILTER_LIST lsst/filter_u.dat,lsst/filter_g.dat,lsst/filter_r.dat,lsst/filter_i.dat,lsst/filter_z.dat,lsst/filter_y.dat +TRANS_TYPE 1 # TRANSMISSION TYPE + # 0[-def]: Energy, 1: Nb of photons +FILTER_CALIB 0,0,0,0,0,0 # 0[-def]: fnu=ctt + # 1 : nu.fnu=ctt + # 2 : fnu=nu + # 3 : fnu=Black Body @ T=10000K + # 4 : for MIPS (leff with nu fnu=ctt and flux with BB @ 10000K +FILTER_FILE filter_lsst # name of file with filters (-> $ZPHOTWORK/filt/) + +# +############################################################################################## +########### THEORETICAL MAGNITUDES ############# +########### Run : $ZPHOTDIR/source/mag_star -c zphot.para (star only) ####### +########### Run : $ZPHOTDIR/source/mag_gal -t (Q or G) -c zphot.para (Gal. & QSO) ####### +############################################################################################## +# +#------------------- From STELLAR LIBRARY ------------------------------ +STAR_LIB_IN LSST_STAR_BIN # Input STELLAR LIBRARY (in $ZPHOTWORK/lib_bin/*) +STAR_LIB_OUT LSST_STAR_MAG # Output STELLAR MAGN. (-> $ZPHOTWORK/lib_mag/*) +# +#------------------- From QSO LIBRARY ------------------------------ +QSO_LIB_IN LSST_QSO_BIN # Input QSO LIBRARY (in $ZPHOTWORK/lib_bin/*) +QSO_LIB_OUT LSST_QSO_MAG # Output QSO MAGN. (-> $ZPHOTWORK/lib_mag/*) +# +#------------------- From GALAXY LIBRARY ------------------------------ +GAL_LIB_IN LSST_GAL_BIN # Input GALAXY LIBRARY (in $ZPHOTWORK/lib_bin/*) +GAL_LIB_OUT LSST_GAL_MAG # Output GALAXY LIBRARY (-> $ZPHOTWORK/lib_mag/*) +# +#------------------ MAG + Z_STEP + COSMO + EXTINCTION ----------------------------- +MAGTYPE AB # Magnitude type (AB or VEGA) +ZGRID_TYPE 0 # Define the kind of redshift grid (0: linear ; 1: dz*(1+z)) +Z_STEP 0.01,0.,7. # dz, zmin, zmax +COSMOLOGY 70,0.3,0.7 # H0,om0,lbd0 (if lb0>0->om0+lbd0=1) +MOD_EXTINC 0,0 # model range for extinction +EXTINC_LAW SB_calzetti.dat # ext. law (in $ZPHOTDIR/ext/*) +EB_V 0.,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.5 # E(B-V) (<50 values) +EM_LINES EMP_UV # [NO/EMP_UV/EMP_SFR/PHYS] choice between emission line prescription +EM_DISPERSION 0.5,0.75,1.,1.5,2. # Dispersion allowed in the emission line flux factor +ADD_DUSTEM NO # Add the dust emission in templates when missing +# +#------------------ ASCII OUTPUT FILES OPTION ------------------------------- +LIB_ASCII NO # Writing Output in ASCII file (-> working directory) +# + + +############################################################################################## +########### PHOTOMETRIC REDSHIFTS ############# +########### Run : $ZPHOTDIR/source/zphot -c zphot.para ############# +############################################################################################## +# +#------------------ Input Catalog Informations ------------------------------- +CAT_IN bidon # Input catalog (full path) +INP_TYPE F # Input type (F:Flux or M:MAG) +CAT_MAG AB # Input Magnitude (AB or VEGA) +CAT_FMT MEME # MEME: (Mag,Err)i + # MMEE: (Mag)i,(Err)i +CAT_LINES 0,1000000000 # MIN and MAX RANGE of ROWS used in input cat [def:-99,-99] +CAT_TYPE LONG # Input Format (LONG,SHORT-def) +GLB_CONTEXT 0 # Overwrite Context (Sum 2^n; n=0->nbd-1, 0->all bands, -1[-def] used context per object) +FORB_CONTEXT -1 # Not consider these filters in the fit (Sum 2^n; n=0->nbd-1) +ERR_SCALE 0.02,0.02,0.02,0.02,0.02,0.02 # Systematic errors per band +ERR_FACTOR 1.5 # Multiply all the flux uncertainties by this factor +ADD_EMLINES 0,10000 # Range of template in which we add emission lines + +# +#------------------ Theoretical libraries -------------------------------- +ZPHOTLIB CE_COSMOS,STAR_COSMOS,QSO_COSMOS # Library used for Chi2 (max:3) + +# +#------------------ Output catalog Name --------------------------------- +CAT_OUT zphot.out # Output catalog (-> working directory) +PARA_OUT $LEPHAREDIR/examples/output.para # Ouput parameters (full path) +VERBOSE NO # Display the templates/objects which are computed +PDZ_TYPE BAY_ZG # PDZ in output [def-BAY]. BAY_ZG sum all probabilities at a given z. MIN_ZG takes exp(-chi2_min/2) at a given z, with chi2_min the minimum chi2 at this z. + # You can add and combine BAY_ZG,BAY_ZQ,MIN_ZG,MIN_ZQ,MASS,SFR,SSFR,AGE +PDZ_OUT test # root of the pdz output files [def-NONE] + # add automatically an extension PDZ_OUT[_zgbay.prob/_zqbay.prob/_zgmin.prob/_zqmin.prob/_mass.prob/_sfr.prob/_age.prob] +# +################# PHOTOMETRIC REDSHIFTS OPTIONS + +#-------------------- Checks ----------------------------------------------------- +RM_DISCREPANT_BD 500 # Threshold in chi2 to consider. Remove <3 bands, stop when below this chi2 threshold + +# +#------------------ Priors --------------------------------- +MAG_ABS -24,-5 # Mabs_min , Mabs_max [0,0-def] +MAG_ABS_QSO -30,-10 # Mabs_min , Mabs_max for QSO library [0,0-def] +MAG_REF 3 # Reference number for band used by Mag_abs +Z_RANGE 0.,99.99 # Z min-max used for the Galaxy library +EBV_RANGE 0,9 # E(B-V) MIN-MAX RANGE of E(B-V) used +#NZ_PRIOR 4,5 # I Band for prior on N(z), the second number is the band to be used if the first is missing. + +# +#------------------ Fixed Z (need format LONG for input Cat) ---------------------- +ZFIX NO # fixed z and search best model [YES,NO-def] +EXTERNALZ_FILE NONE # Fix the redshift from an external file. + +# +#------------------ Parabolic interpolation for Zbest ---------------------------- +Z_INTERP YES # redshift interpolation (for GAL and QSO librairies) +# +Z_METHOD BEST # BEST/ML absolute magnitude, .spec, scaling will be given at this redshift + +#------------------ Normalized ML(exp-(0.5*Chi^2)) curve analysis ------------------- +#------- Secondary peak analysis +DZ_WIN 1.0 # Window search for 2nd peaks [0->5; 0.25-def] +MIN_THRES 0.02 # Lower theshold for 2nd peaks [0->1; 0.1-def] +# +#------- Probability (in %) per redshift intervals -------------------- +#PROB_INTZ 0.01,0.1,0.1,0.2,0.2,0.3,0.3,0.4,0.4,0.5,0.5,0.6,0.6,0.7,0.7,0.8,0.8,0.9,0.9,1.,1.,1.1,1.1,1.3,1.3,1.5,1.5,2.,2.,3.,3.,4.,4.,5 # (even number) +#PDZ_MABS_FILT 2,10,14 # MABS for REF FILTERS to be extracted + + +#--------------------- OUTPUT SPECTRA -------------------------------------- +# +SPEC_OUT NO # spectrum for each object? [YES,NO-def] +CHI2_OUT NO # output file with all values : z,mod,chi2,E(B-V),... + # BE CAREFUL can take a lot of space !! + + +#------------------ MAGNITUDE SHIFTS applied to libraries ---------------------------- +# +# APPLY_SYSSHIFT 0. # Apply systematic shifts in each band + # used only if number of shifts matches + # with number of filters in the library + +# +#------------------- ADAPTIVE METHOD using Z spectro sample ----------------- +# +AUTO_ADAPT NO # Adapting method with spectro [NO-def] +ADAPT_BAND 5 # Reference band, band1, band2 for color +ADAPT_LIM 1.5,23.0 # Mag limits for spectro in Ref band [18,21.5-def] +ADAPT_CONTEXT -1 # Context for bands used for training + # If -1[-def] the ones given by the normal context +ADAPT_ZBIN 0.01,6 # Redshift's interval used for training + # [0.001,6-Def] +ADAPT_MODBIN 1,1000 # Model's interval used for training + # [1,1000-Def] + + +################## ADDITION OPTION OF ZPHOTA FOR PHYSICAL PARAMETERS + +# +#-------- FIR LIBRARY+FIT ---------------------------------------------------- +FIR_LIB NONE +FIR_LMIN 7.0 # Lambda Min (micron) for FIR analysis +FIR_CONT -1 +FIR_SCALE -1 +FIR_FREESCALE YES # ALLOW FOR FREE SCALING +FIR_SUBSTELLAR NO + + +# +#--------------- ABSOLUTE MAGNITUDES COMPUTATION --------------------------- +# +MABS_METHOD 1 # 0[-def] : obs->Ref + # 1 : best obs->Ref + # 2 : fixed obs->Ref + # 3 : mag from best SED + # 4 : Zbin +MABS_CONTEXT 63 + +MABS_REF 1 # 0[-def]: filter obs chosen for Mabs : + # ONLY USED IF MABS_METHOD=2 +MABS_FILT 1,2,3,4 # Chosen filters per redshift bin (MABS_ZBIN) + # ONLY USED IF MABS_METHOD=4 +MABS_ZBIN 0,0.5,2,4,6 # Redshift bins corresponding to MABS_FILT (1st bin 0,0.5, second 0.5,2 ...) + # ONLY USED IF MABS_METHOD=4 +RF_COLORS 32,4,4,13 # When computing uncertainties on abs. mag., possible to do it in two colors +ADDITIONAL_MAG none # name of file with filters (-> $ZPHOTWORK/filt/). Predicted mag and abs mag in additional filters. + + +#------------------- Z MAX ----------------------------------------------------------- + +LIMITS_ZBIN 0,99 # Redshifts used to split in N bins, separated by a coma. Need N+1 values (start with the minimum redshift). +LIMITS_MAPP_REF 1 # Band in which the absolute magnitude is computed +LIMITS_MAPP_SEL 1 # Give the selection band in each redshift bin. Need 1 or N values. +LIMITS_MAPP_CUT 90 # Magnitude cut used in each redshift bin. Need 1 or N values. From 3ef0197d80ece47896c34be7c5c905f6ad62debd Mon Sep 17 00:00:00 2001 From: Raphael Shirley Date: Mon, 25 Mar 2024 21:43:32 +0100 Subject: [PATCH 3/3] Fix os import bug --- src/rail/estimation/algos/lephare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rail/estimation/algos/lephare.py b/src/rail/estimation/algos/lephare.py index 482c379..0eced6d 100644 --- a/src/rail/estimation/algos/lephare.py +++ b/src/rail/estimation/algos/lephare.py @@ -1,6 +1,7 @@ # import qp from rail.estimation.estimator import CatEstimator, CatInformer from rail.core.common_params import SHARED_PARAMS +import os import lephare as lp