From efba842092f60b43c0b05012daa136b72f417378 Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Sun, 15 Mar 2020 15:11:29 -0700 Subject: [PATCH 01/12] First Commit: This code has the function that generates the automated validation table. Next iteration will have validation by eye function to check automated table to a master version. --- Metallicity_Stack_Commons/valid_table.py | 105 +++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 Metallicity_Stack_Commons/valid_table.py diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py new file mode 100644 index 0000000..6cce160 --- /dev/null +++ b/Metallicity_Stack_Commons/valid_table.py @@ -0,0 +1,105 @@ +import numpy as np +import matplotlib.pyplot as plt +from astropy.io import fits +from astropy.io import ascii as asc +from astropy.table import Table + +from Metallicity_Stack_Commons.column_names import filename_dict, bin_names0 + +def make_validation_table(fitspath): #bin_type_str + ''' + Purpose: + This function creates a validation table for a given binning set. The validation table + contains a OIII4363 detection column where 1.0 means detection, 0.5 means non-detection with + reliable OIII5007, and 0.0 means unreliable non-detection. + This function will be run every time the analysis is completed and will create a validation + table for every analysis. + + Usage: + valid_table.make_validation_table(fitspath, bin_type_str) + + Params: + fitspath --> a string of the file path where the input file is and where the output file + will be placed. + not in use: + bin_type_str --> a string describing the binning type. (e.g. 'massLHbetabin' or 'massbin') + --> This is dataset for Zcalbase_gal analysis + + Returns: + None + + Outputs: + fitspath + 'validation.tbl' --> a validation table containing bin IDs; + number of galaxies in each bin; and + column indicating OIII4363 detection/non-detection + OIII4363_Flux_Observed + OIII4363_S/N + ''' + + em_table = asc.read(fitspath + filename_dict['bin_fit']) #this is combine_flux_ascii + + bin_ID = em_table['bin_ID'].data + raw_OIII4363 = em_table['OIII_4363_Flux_Observed'].data + O_4363_SN = em_table['OIII_4363_S/N'].data + O_4363_sigma = em_table['OIII_4363_Sigma'].data + O_5007_SN = em_table['OIII_5007_S/N'].data + + N_stack = em_table['N_stack'].data + Hgamma_SN = em_table['HGAMMA_S/N'].data + Hgamma = em_table['HGAMMA_Flux_Observed'].data + + detection = np.zeros(len(bin_ID)) + OIII4363 = np.zeros(len(bin_ID)) + up_limit = (Hgamma/Hgamma_SN) *3 + + + valid_stacks_idx = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma < 2))[0] + reliable_5007_stacks = np.where((O_4363_SN < 3) & (O_5007_SN > 100))[0] + wide_lines_valid = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma >= 2))[0] + detection[valid_stacks_idx] = 1 + detection[reliable_5007_stacks] = 0.5 + detection[wide_lines_valid] = 0.5 + print(detection) + + for ii in range(len(OIII4363)): + + if detection[ii] == 1: + OIII4363[ii]= raw_OIII4363[ii] + if detection[ii] == 0.5: + OIII4363[ii]= up_limit[ii] + if detection[ii] ==0: + OIII4363[ii]= up_limit[ii] + + ver_tab = fitspath+ filename_dict['bin_valid'] + n =('bin_ID','N_stack','Detection', 'OIII_4363_Flux_Observed', 'OIII_4363_S/N') + tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names=n) + asc.write(tab1, ver_tab, format='fixed_width_two_line') + +def quality_assurance(bin_type_str, QA_flag): + ''' + Purpose: + This function allows for manual flagging of sources for quality assurance of OIII4363 detections + for the mass luminosity analysis. Based on the bin_type_str keyword, the user can override the + detection flag by setting a specificbin index equal to the desired flag (1.0, 0.5, or 0.0). + + Usage: + valid_table.quality_assurance(bin_type_str, QA_flag) + + Params: + bin_type_str --> a string describing the binning type. (e.g. 'massLHbetabin' or 'massbin') + QA_flag --> a numpy zeros array the size of the number of bins. This is used to flag sources by + changing the value at a specific index to the desired flag. + + Returns: + QA_flag --> the updated flag array. + + Outputs: + None + ''' + if bin_type_str == 'mass_LHbeta_bin': + QA_flag[10] = 1.0 #has large line width on OIII4363 + QA_flag[11] = 1.0 #has large line width on OIII4363 + elif bin_type_str == 'massbin': + QA_flag[5] = 1.0 #has large line width on OIII4363 + + return QA_flag From a9d828ee92fb588d9a9cc83a74fadf409549bdcf Mon Sep 17 00:00:00 2001 From: Caroline McCormick Date: Sun, 15 Mar 2020 15:44:54 -0700 Subject: [PATCH 02/12] (Issue #13) Added bin info table to get N_stack values. Got rid of quality assurance function. --- Metallicity_Stack_Commons/valid_table.py | 34 +++--------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index 6cce160..c3f0b55 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -6,7 +6,7 @@ from Metallicity_Stack_Commons.column_names import filename_dict, bin_names0 -def make_validation_table(fitspath): #bin_type_str +def make_validation_table(fitspath): ''' Purpose: This function creates a validation table for a given binning set. The validation table @@ -36,6 +36,7 @@ def make_validation_table(fitspath): #bin_type_str OIII4363_S/N ''' + bin_table = asc.read(fitspath + filename_dict['bin_info']) em_table = asc.read(fitspath + filename_dict['bin_fit']) #this is combine_flux_ascii bin_ID = em_table['bin_ID'].data @@ -44,7 +45,7 @@ def make_validation_table(fitspath): #bin_type_str O_4363_sigma = em_table['OIII_4363_Sigma'].data O_5007_SN = em_table['OIII_5007_S/N'].data - N_stack = em_table['N_stack'].data + N_stack = bin_table['N_stack'].data Hgamma_SN = em_table['HGAMMA_S/N'].data Hgamma = em_table['HGAMMA_Flux_Observed'].data @@ -75,31 +76,4 @@ def make_validation_table(fitspath): #bin_type_str tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names=n) asc.write(tab1, ver_tab, format='fixed_width_two_line') -def quality_assurance(bin_type_str, QA_flag): - ''' - Purpose: - This function allows for manual flagging of sources for quality assurance of OIII4363 detections - for the mass luminosity analysis. Based on the bin_type_str keyword, the user can override the - detection flag by setting a specificbin index equal to the desired flag (1.0, 0.5, or 0.0). - - Usage: - valid_table.quality_assurance(bin_type_str, QA_flag) - - Params: - bin_type_str --> a string describing the binning type. (e.g. 'massLHbetabin' or 'massbin') - QA_flag --> a numpy zeros array the size of the number of bins. This is used to flag sources by - changing the value at a specific index to the desired flag. - - Returns: - QA_flag --> the updated flag array. - - Outputs: - None - ''' - if bin_type_str == 'mass_LHbeta_bin': - QA_flag[10] = 1.0 #has large line width on OIII4363 - QA_flag[11] = 1.0 #has large line width on OIII4363 - elif bin_type_str == 'massbin': - QA_flag[5] = 1.0 #has large line width on OIII4363 - - return QA_flag + From 94996a12ed5e686b9b638a47c276efbfbdbaa251 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Tue, 17 Mar 2020 16:21:08 -0700 Subject: [PATCH 03/12] Update column_names from develop branch (iss #25) --- Metallicity_Stack_Commons/column_names.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Metallicity_Stack_Commons/column_names.py b/Metallicity_Stack_Commons/column_names.py index 3ad850f..5f1b934 100644 --- a/Metallicity_Stack_Commons/column_names.py +++ b/Metallicity_Stack_Commons/column_names.py @@ -1,4 +1,8 @@ from . import line_name, line_type +import sys + +# Get python version +py_vers = sys.version_info.major # Need to define here @@ -46,7 +50,7 @@ def line_fit_suffix_add(line_name0, line_type0): # Column names for Gaussian fitting # This is just the suffix gauss_names0 = ['Flux_Gaussian', 'Flux_Observed', 'S/N', 'Center', 'Norm', - 'Median', 'Sigma'] + 'Median', 'Sigma', 'RMS'] balmer_names0 = ['Abs_Norm', 'Abs_Sigma'] # Emission-line fit column names with [LINE] prefix @@ -106,7 +110,10 @@ def remove_from_list(list0, remove_entries): :param remove_entries: list of column names to remove """ - dup_list0 = list0.copy() + if py_vers == 3: + dup_list0 = list0.copy() + if py_vers == 2: + dup_list0 = list(list0) for entry in remove_entries: dup_list0.remove(entry) From 04f1feb51ab2081da71d480719b9e0eaccc8296e Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Thu, 19 Mar 2020 16:04:01 -0700 Subject: [PATCH 04/12] Added a validation_table to the column names --- Metallicity_Stack_Commons/column_names.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Metallicity_Stack_Commons/column_names.py b/Metallicity_Stack_Commons/column_names.py index 5f1b934..2d4efa1 100644 --- a/Metallicity_Stack_Commons/column_names.py +++ b/Metallicity_Stack_Commons/column_names.py @@ -62,6 +62,9 @@ def line_fit_suffix_add(line_name0, line_type0): # Temperature and metallicity properties temp_metal_names0 = ['T_e', '12+log(O/H)', 'log(O+/H)', 'log(O++/H)', 'O+/H', 'O++/H'] +# Validation Table +valid_table_names0 = ['bin_ID','N_stack','Detection', 'OIII_4363_Flux_Observed', 'OIII_4363_S/N'] + # Dictionary containing filenames filename_dict = dict() From 7bb378905eff9ced08d8d7cd11c08cc187e782a5 Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Thu, 19 Mar 2020 16:07:11 -0700 Subject: [PATCH 05/12] Added a function that checks the automated validation table to a make a revised table --- Metallicity_Stack_Commons/valid_table.py | 90 ++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index c3f0b55..d1589b0 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -2,9 +2,9 @@ import matplotlib.pyplot as plt from astropy.io import fits from astropy.io import ascii as asc -from astropy.table import Table +from astropy.table import Table, Column -from Metallicity_Stack_Commons.column_names import filename_dict, bin_names0 +from Metallicity_Stack_Commons.column_names import filename_dict, bin_names0, remove_from_list def make_validation_table(fitspath): ''' @@ -29,7 +29,7 @@ def make_validation_table(fitspath): None Outputs: - fitspath + 'validation.tbl' --> a validation table containing bin IDs; + fitspath + 'bin_validation.tbl' --> a validation table containing bin IDs; number of galaxies in each bin; and column indicating OIII4363 detection/non-detection OIII4363_Flux_Observed @@ -37,7 +37,7 @@ def make_validation_table(fitspath): ''' bin_table = asc.read(fitspath + filename_dict['bin_info']) - em_table = asc.read(fitspath + filename_dict['bin_fit']) #this is combine_flux_ascii + em_table = asc.read(fitspath + filename_dict['bin_fit']) bin_ID = em_table['bin_ID'].data raw_OIII4363 = em_table['OIII_4363_Flux_Observed'].data @@ -73,7 +73,87 @@ def make_validation_table(fitspath): ver_tab = fitspath+ filename_dict['bin_valid'] n =('bin_ID','N_stack','Detection', 'OIII_4363_Flux_Observed', 'OIII_4363_S/N') - tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names=n) + tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names = valid_table_names0) #names=n) asc.write(tab1, ver_tab, format='fixed_width_two_line') +def compare_to_by_eye(fitspath,dataset): + ''' + Purpose -> This function takes the automated validation table and checks it against + inputted measurement that are determined by eye. These inputted measurements + are in the np.where statements. It outputs a revised validation table based + on the inputted measurements. + + Usage -> valid_table.make_validation_table(fitspath, dataset) + + Params: + fitspath --> a string of the file path where the input file is and where the output file + will be placed + dataset --> a string that is used to determine which eye measurements to use + + Returns: + None + + Outputs: + fitspath + 'bin_validation_revised.tbl' and '.csv' --> + a validation table containing bin IDs; + number of galaxies in each bin; and + column indicating OIII4363 detection/non-detection + OIII4363_Flux_Observed + OIII4363_S/N + Notes + + ''' + ver_table = fitspath+ filename_dict['bin_valid'] + ver_tab = asc.read(ver_table) + indicate = ver_tab['Detection'] + ID = ver_tab['bin_ID'] + + #Detections By Eye + if dataset== 'Voronoi20': det_4363 = np.where((ID ==0)| (ID ==2)| (ID ==3) | (ID ==5) | (ID==6))[0] + if dataset== 'Voronoi14': det_4363 = np.where((ID ==0)| (ID ==7)| (ID ==10) | (ID ==11) | (ID==12))[0] + if dataset== 'Voronoi10': det_4363 = np.where((ID ==1)| (ID ==9)| (ID ==18) | (ID ==21))[0] + if dataset== 'Grid': det_4363 = np.where((ID ==11)| (ID ==13)| (ID ==19) | (ID ==20) | (ID == 21))[0] + if dataset== 'R23_Grid': det_4363 = np.where((ID ==0)| (ID ==4)| (ID ==5) | (ID ==6))[0] + if dataset== 'O32_Grid': det_4363 = np.where((ID ==6))[0] + if dataset== 'Double_Bin': det_4363 = np.where((ID ==0)| (ID ==1)| (ID ==2) | (ID ==7) | (ID ==9) | (ID ==10) | (ID ==11) | (ID ==13))[0] + if dataset== 'n_Bins': + det_4363= np.where( (ID ==10) | (ID ==11) |(ID==14) | (ID ==15) | (ID ==20) | (ID==23) | (ID ==26))[0] + rlimit = np.where( (ID ==5) | (ID ==7) |(ID==8) | (ID ==13) | (ID ==16) | (ID==17) | (ID ==19)| (ID ==22))[0] + + ######Caroline: Add you conditions here####### + + check_ID = np.zeros(len(ID)) + + check_ID[det_4363]=1 + check_ID[rlimit] =0.5 + + for ii in range(len(ID)): + if check_ID[ii] == indicate[ii]: + print(ID[ii], 'matches with by eye validation') + else: + print('*****', ID[ii], 'does not match calculated values. Please check!') + + ###This is where I need to add the column for notes + if dataset == 'n_Bins': + notes = ['N/A','N/A','N/A','N/A','N/A', + 'N/A','N/A','Broad features, but reliable OIII5007 and HGAMMA', + 'Bad fit, but good OIII5007','N/A', + 'N/A','N/A','N/A','N/A','N/A', + 'High Temperature','not fit well, but reliable OIII5007 and HGAMMA', + 'N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A'] + note_add = Column(name='Notes', data = notes) + ver_tab.add_column(note_add,5) + + #######Caroline: Add your notes column here and copy the note_add and ver_tab.add_column lines to your if statement###### + + + ver_tab.remove_column('Detection') + + detect_add = Column(name='Detections', data =check_ID) + ver_tab.add_column(detect_add, 2) + + asc.write(ver_tab, fitspath+ 'bin_validation_revised.tbl', format = 'fixed_width_two_line') + asc.write(ver_tab, fitspath+ 'bin_validation_revised.csv', format = 'csv') + + From 5d1db9bed04647bac116b1e38695170b5264561d Mon Sep 17 00:00:00 2001 From: Caroline McCormick Date: Thu, 26 Mar 2020 08:36:05 -0700 Subject: [PATCH 06/12] (Issue #13) Changed EW from 2Ang to 1.6Ang. --- Metallicity_Stack_Commons/valid_table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index d1589b0..58f2013 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -54,9 +54,9 @@ def make_validation_table(fitspath): up_limit = (Hgamma/Hgamma_SN) *3 - valid_stacks_idx = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma < 2))[0] + valid_stacks_idx = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma < 1.6))[0] reliable_5007_stacks = np.where((O_4363_SN < 3) & (O_5007_SN > 100))[0] - wide_lines_valid = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma >= 2))[0] + wide_lines_valid = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma >= 1.6))[0] detection[valid_stacks_idx] = 1 detection[reliable_5007_stacks] = 0.5 detection[wide_lines_valid] = 0.5 From 13bfb9150451c6fcd912def0a6de6cfa2b03fe49 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Thu, 26 Mar 2020 08:55:19 -0700 Subject: [PATCH 07/12] Write revised valid table in valid_table.make_validation_table; Update column_names to include bin_valid_rev in filenames_dict (iss #13, #25) --- Metallicity_Stack_Commons/column_names.py | 1 + Metallicity_Stack_Commons/valid_table.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/Metallicity_Stack_Commons/column_names.py b/Metallicity_Stack_Commons/column_names.py index 2d4efa1..62bb685 100644 --- a/Metallicity_Stack_Commons/column_names.py +++ b/Metallicity_Stack_Commons/column_names.py @@ -71,6 +71,7 @@ def line_fit_suffix_add(line_name0, line_type0): # Bin-related files filename_dict['bin_info'] = 'bin_info.tbl' filename_dict['bin_valid'] = 'bin_validation.tbl' +filename_dict['bin_valid_rev'] = filename_dict['bin_valid'].replace('.tbl', '.revised.tbl') filename_dict['bin_fit'] = 'bin_emission_line_fit.tbl' filename_dict['bin_fit_rev'] = filename_dict['bin_fit'].replace('.tbl', '.revised.tbl') filename_dict['bin_derived_prop'] = 'bin_derived_properties.tbl' diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index d1589b0..cb5f6d9 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -1,3 +1,4 @@ +from os.path import exists import numpy as np import matplotlib.pyplot as plt from astropy.io import fits @@ -76,6 +77,19 @@ def make_validation_table(fitspath): tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names = valid_table_names0) #names=n) asc.write(tab1, ver_tab, format='fixed_width_two_line') + # Write revised file for human editing + ver_tab_revised = fitspath + filename_dict['bin_valid_rev'] + if not exists(ver_tab_revised): + asc.write(tab1, ver_tab_revised, format='fixed_width_two_line') + print(" ") + print("URGENT!!! HUMAN EDITING OF FILE NEEDED : "+ver_tab_revised) + print(" ") + else: + print(" ") + print("ERROR!!! FILE EXISTS!!! WILL NOT OVERWRITE !!!") + print("ERROR!!! PLEASE RENAME/DELETE FILE TO REGENERATE !!!") + print(" ") + def compare_to_by_eye(fitspath,dataset): ''' From a564367a6f00be3c71274f678d1a85d98c99d149 Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Thu, 26 Mar 2020 09:07:08 -0700 Subject: [PATCH 08/12] valid_table: PEP8 fixes and documentation (#13) --- Metallicity_Stack_Commons/valid_table.py | 152 ++++++++++++----------- 1 file changed, 78 insertions(+), 74 deletions(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index af1c979..b967a24 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -1,41 +1,40 @@ from os.path import exists import numpy as np -import matplotlib.pyplot as plt -from astropy.io import fits from astropy.io import ascii as asc from astropy.table import Table, Column -from Metallicity_Stack_Commons.column_names import filename_dict, bin_names0, remove_from_list +from Metallicity_Stack_Commons.column_names import filename_dict, valid_table_names0 # , bin_names0, remove_from_list + def make_validation_table(fitspath): - ''' - Purpose: + """ + Purpose: This function creates a validation table for a given binning set. The validation table contains a OIII4363 detection column where 1.0 means detection, 0.5 means non-detection with reliable OIII5007, and 0.0 means unreliable non-detection. - This function will be run every time the analysis is completed and will create a validation - table for every analysis. - + This function will be run every time the analysis is completed and will create a validation + table for every analysis. + Usage: valid_table.make_validation_table(fitspath, bin_type_str) - + Params: fitspath --> a string of the file path where the input file is and where the output file will be placed. - not in use: - bin_type_str --> a string describing the binning type. (e.g. 'massLHbetabin' or 'massbin') + not in use: + bin_type_str --> a string describing the binning type. (e.g. 'massLHbetabin' or 'massbin') --> This is dataset for Zcalbase_gal analysis - - Returns: + + Returns: None - + Outputs: fitspath + 'bin_validation.tbl' --> a validation table containing bin IDs; - number of galaxies in each bin; and + number of galaxies in each bin; and column indicating OIII4363 detection/non-detection OIII4363_Flux_Observed OIII4363_S/N - ''' + """ bin_table = asc.read(fitspath + filename_dict['bin_info']) em_table = asc.read(fitspath + filename_dict['bin_fit']) @@ -47,13 +46,12 @@ def make_validation_table(fitspath): O_5007_SN = em_table['OIII_5007_S/N'].data N_stack = bin_table['N_stack'].data - Hgamma_SN = em_table['HGAMMA_S/N'].data + Hgamma_SN = em_table['HGAMMA_S/N'].data Hgamma = em_table['HGAMMA_Flux_Observed'].data detection = np.zeros(len(bin_ID)) OIII4363 = np.zeros(len(bin_ID)) - up_limit = (Hgamma/Hgamma_SN) *3 - + up_limit = (Hgamma/Hgamma_SN) * 3 valid_stacks_idx = np.where((O_4363_SN >= 3) & (O_5007_SN > 100) & (O_4363_sigma < 1.6))[0] reliable_5007_stacks = np.where((O_4363_SN < 3) & (O_5007_SN > 100))[0] @@ -64,17 +62,15 @@ def make_validation_table(fitspath): print(detection) for ii in range(len(OIII4363)): - - if detection[ii] == 1: - OIII4363[ii]= raw_OIII4363[ii] - if detection[ii] == 0.5: - OIII4363[ii]= up_limit[ii] - if detection[ii] ==0: - OIII4363[ii]= up_limit[ii] - - ver_tab = fitspath+ filename_dict['bin_valid'] - n =('bin_ID','N_stack','Detection', 'OIII_4363_Flux_Observed', 'OIII_4363_S/N') - tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names = valid_table_names0) #names=n) + if detection[ii] == 1: + OIII4363[ii] = raw_OIII4363[ii] + if detection[ii] == 0.5: + OIII4363[ii] = up_limit[ii] + if detection[ii] == 0: + OIII4363[ii] = up_limit[ii] + + ver_tab = fitspath + filename_dict['bin_valid'] + tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names=valid_table_names0) asc.write(tab1, ver_tab, format='fixed_width_two_line') # Write revised file for human editing @@ -91,56 +87,67 @@ def make_validation_table(fitspath): print(" ") -def compare_to_by_eye(fitspath,dataset): - ''' - Purpose -> This function takes the automated validation table and checks it against - inputted measurement that are determined by eye. These inputted measurements - are in the np.where statements. It outputs a revised validation table based - on the inputted measurements. - +def compare_to_by_eye(fitspath, dataset): + """ + Purpose -> This function takes the automated validation table and checks it against + inputted measurement that are determined by eye. These inputted measurements + are in the np.where statements. It outputs a revised validation table based + on the inputted measurements. + Usage -> valid_table.make_validation_table(fitspath, dataset) Params: fitspath --> a string of the file path where the input file is and where the output file will be placed dataset --> a string that is used to determine which eye measurements to use - - Returns: + + Returns: None - + Outputs: - fitspath + 'bin_validation_revised.tbl' and '.csv' --> + fitspath + 'bin_validation_revised.tbl' and '.csv' --> a validation table containing bin IDs; - number of galaxies in each bin; and + number of galaxies in each bin; and column indicating OIII4363 detection/non-detection OIII4363_Flux_Observed OIII4363_S/N Notes - ''' - ver_table = fitspath+ filename_dict['bin_valid'] + """ + ver_table = fitspath + filename_dict['bin_valid'] ver_tab = asc.read(ver_table) indicate = ver_tab['Detection'] ID = ver_tab['bin_ID'] - #Detections By Eye - if dataset== 'Voronoi20': det_4363 = np.where((ID ==0)| (ID ==2)| (ID ==3) | (ID ==5) | (ID==6))[0] - if dataset== 'Voronoi14': det_4363 = np.where((ID ==0)| (ID ==7)| (ID ==10) | (ID ==11) | (ID==12))[0] - if dataset== 'Voronoi10': det_4363 = np.where((ID ==1)| (ID ==9)| (ID ==18) | (ID ==21))[0] - if dataset== 'Grid': det_4363 = np.where((ID ==11)| (ID ==13)| (ID ==19) | (ID ==20) | (ID == 21))[0] - if dataset== 'R23_Grid': det_4363 = np.where((ID ==0)| (ID ==4)| (ID ==5) | (ID ==6))[0] - if dataset== 'O32_Grid': det_4363 = np.where((ID ==6))[0] - if dataset== 'Double_Bin': det_4363 = np.where((ID ==0)| (ID ==1)| (ID ==2) | (ID ==7) | (ID ==9) | (ID ==10) | (ID ==11) | (ID ==13))[0] - if dataset== 'n_Bins': - det_4363= np.where( (ID ==10) | (ID ==11) |(ID==14) | (ID ==15) | (ID ==20) | (ID==23) | (ID ==26))[0] - rlimit = np.where( (ID ==5) | (ID ==7) |(ID==8) | (ID ==13) | (ID ==16) | (ID==17) | (ID ==19)| (ID ==22))[0] + # Detections By Eye + if dataset == 'Voronoi20': + det_4363 = np.where((ID == 0) | (ID == 2) | (ID == 3) | (ID == 5) | (ID == 6))[0] + if dataset == 'Voronoi14': + det_4363 = np.where((ID == 0) | (ID == 7) | (ID == 10) | (ID == 11) | (ID == 12))[0] + if dataset == 'Voronoi10': + det_4363 = np.where((ID == 1) | (ID == 9) | (ID == 18) | (ID == 21))[0] + if dataset == 'Grid': + det_4363 = np.where((ID == 11) | (ID == 13) | (ID == 19) | (ID == 20) | (ID == 21))[0] + if dataset == 'R23_Grid': + det_4363 = np.where((ID == 0) | (ID == 4) | (ID == 5) | (ID == 6))[0] + if dataset == 'O32_Grid': + det_4363 = np.where((ID == 6))[0] + if dataset == 'Double_Bin': + det_4363 = np.where((ID == 0) | (ID == 1) | (ID == 2) | (ID == 7) | (ID == 9) | + (ID == 10) | (ID == 11) | (ID == 13))[0] + if dataset == 'n_Bins': + det_4363 = np.where((ID == 10) | (ID == 11) | (ID == 14) | (ID == 15) | (ID == 20) | + (ID == 23) | (ID == 26))[0] + rlimit = np.where((ID == 5) | (ID == 7) | (ID == 8) | (ID == 13) | (ID == 16) | + (ID == 17) | (ID == 19) | (ID == 22))[0] - ######Caroline: Add you conditions here####### + # Caroline: Add you conditions here check_ID = np.zeros(len(ID)) - check_ID[det_4363]=1 - check_ID[rlimit] =0.5 + check_ID[det_4363] = 1 + if dataset == 'n_Bins': + check_ID[rlimit] = 0.5 for ii in range(len(ID)): if check_ID[ii] == indicate[ii]: @@ -148,26 +155,23 @@ def compare_to_by_eye(fitspath,dataset): else: print('*****', ID[ii], 'does not match calculated values. Please check!') - ###This is where I need to add the column for notes + # This is where I need to add the column for notes if dataset == 'n_Bins': - notes = ['N/A','N/A','N/A','N/A','N/A', - 'N/A','N/A','Broad features, but reliable OIII5007 and HGAMMA', - 'Bad fit, but good OIII5007','N/A', - 'N/A','N/A','N/A','N/A','N/A', - 'High Temperature','not fit well, but reliable OIII5007 and HGAMMA', - 'N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A','N/A'] - note_add = Column(name='Notes', data = notes) - ver_tab.add_column(note_add,5) + notes = ['N/A', 'N/A', 'N/A', 'N/A', 'N/A', + 'N/A', 'N/A', 'Broad features, but reliable OIII5007 and HGAMMA', + 'Bad fit, but good OIII5007', 'N/A', + 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', + 'High Temperature', 'not fit well, but reliable OIII5007 and HGAMMA', + 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A'] + note_add = Column(name='Notes', data=notes) + ver_tab.add_column(note_add, 5) - #######Caroline: Add your notes column here and copy the note_add and ver_tab.add_column lines to your if statement###### + # Caroline: Add your notes column here and copy the note_add and ver_tab.add_column lines to your if statement - ver_tab.remove_column('Detection') - detect_add = Column(name='Detections', data =check_ID) + detect_add = Column(name='Detections', data=check_ID) ver_tab.add_column(detect_add, 2) - asc.write(ver_tab, fitspath+ 'bin_validation_revised.tbl', format = 'fixed_width_two_line') - asc.write(ver_tab, fitspath+ 'bin_validation_revised.csv', format = 'csv') - - + asc.write(ver_tab, fitspath + 'bin_validation_revised.tbl', format='fixed_width_two_line') + asc.write(ver_tab, fitspath + 'bin_validation_revised.csv', format='csv') From 054221eec7da5ef3e2e65958a0628da37190342d Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Sat, 28 Mar 2020 22:58:54 -0700 Subject: [PATCH 09/12] Fixed a table column name from Detections->Detection --- Metallicity_Stack_Commons/valid_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index b967a24..e797117 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -170,7 +170,7 @@ def compare_to_by_eye(fitspath, dataset): ver_tab.remove_column('Detection') - detect_add = Column(name='Detections', data=check_ID) + detect_add = Column(name='Detection', data=check_ID) ver_tab.add_column(detect_add, 2) asc.write(ver_tab, fitspath + 'bin_validation_revised.tbl', format='fixed_width_two_line') From 4f3d6775f28eedf68ce15e30aa5e5ccc90f3fd08 Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Sat, 11 Apr 2020 18:14:55 -0700 Subject: [PATCH 10/12] Fixed import k_dict call because it is now in the directory above --- Metallicity_Stack_Commons/analysis/temp_metallicity_calc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Metallicity_Stack_Commons/analysis/temp_metallicity_calc.py b/Metallicity_Stack_Commons/analysis/temp_metallicity_calc.py index b4312a1..1302b7f 100644 --- a/Metallicity_Stack_Commons/analysis/temp_metallicity_calc.py +++ b/Metallicity_Stack_Commons/analysis/temp_metallicity_calc.py @@ -1,6 +1,6 @@ import numpy as np -from . import k_dict +from .. import k_dict from ..column_names import temp_metal_names0, remove_from_list From 20027639122812cdfbab40f9d35693aec7ac617d Mon Sep 17 00:00:00 2001 From: Reagen Leimbach Date: Sat, 11 Apr 2020 20:21:21 -0700 Subject: [PATCH 11/12] Updated valid_table to be used in Zcalbase_gal and added revised option to column_name for valid_table --- Metallicity_Stack_Commons/column_names.py | 1 + Metallicity_Stack_Commons/valid_table.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Metallicity_Stack_Commons/column_names.py b/Metallicity_Stack_Commons/column_names.py index f191615..757cd2e 100644 --- a/Metallicity_Stack_Commons/column_names.py +++ b/Metallicity_Stack_Commons/column_names.py @@ -71,6 +71,7 @@ def line_fit_suffix_add(line_name0, line_type0): # Bin-related files filename_dict['bin_info'] = 'bin_info.tbl' filename_dict['bin_valid'] = 'bin_validation.tbl' +filename_dict['bin_valid_rev'] = 'bin_validation_revised.tbl' filename_dict['bin_valid_rev'] = filename_dict['bin_valid'].replace('.tbl', '.revised.tbl') filename_dict['bin_fit'] = 'bin_emission_line_fit.tbl' filename_dict['bin_fit_rev'] = filename_dict['bin_fit'].replace('.tbl', '.revised.tbl') diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index e797117..b2ad29b 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -3,7 +3,7 @@ from astropy.io import ascii as asc from astropy.table import Table, Column -from Metallicity_Stack_Commons.column_names import filename_dict, valid_table_names0 # , bin_names0, remove_from_list +from Metallicity_Stack_Commons.Metallicity_Stack_Commons.column_names import filename_dict, valid_table_names0 # , bin_names0, remove_from_list def make_validation_table(fitspath): @@ -72,7 +72,7 @@ def make_validation_table(fitspath): ver_tab = fitspath + filename_dict['bin_valid'] tab1 = Table([bin_ID, N_stack, detection, OIII4363, O_4363_SN], names=valid_table_names0) asc.write(tab1, ver_tab, format='fixed_width_two_line') - + ''' # Write revised file for human editing ver_tab_revised = fitspath + filename_dict['bin_valid_rev'] if not exists(ver_tab_revised): @@ -85,7 +85,7 @@ def make_validation_table(fitspath): print("ERROR!!! FILE EXISTS!!! WILL NOT OVERWRITE !!!") print("ERROR!!! PLEASE RENAME/DELETE FILE TO REGENERATE !!!") print(" ") - + ''' def compare_to_by_eye(fitspath, dataset): """ @@ -173,5 +173,5 @@ def compare_to_by_eye(fitspath, dataset): detect_add = Column(name='Detection', data=check_ID) ver_tab.add_column(detect_add, 2) - asc.write(ver_tab, fitspath + 'bin_validation_revised.tbl', format='fixed_width_two_line') + asc.write(ver_tab, fitspath + filename_dict['bin_valid_rev'], format='fixed_width_two_line') asc.write(ver_tab, fitspath + 'bin_validation_revised.csv', format='csv') From ccc9ad289037b2cb35bc255612b86aba656d03ce Mon Sep 17 00:00:00 2001 From: Chun Ly Date: Sun, 12 Apr 2020 20:28:00 -0700 Subject: [PATCH 12/12] valid_table: Use relative import for column_names (iss #13) --- Metallicity_Stack_Commons/valid_table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Metallicity_Stack_Commons/valid_table.py b/Metallicity_Stack_Commons/valid_table.py index b2ad29b..7c4eda3 100644 --- a/Metallicity_Stack_Commons/valid_table.py +++ b/Metallicity_Stack_Commons/valid_table.py @@ -3,7 +3,7 @@ from astropy.io import ascii as asc from astropy.table import Table, Column -from Metallicity_Stack_Commons.Metallicity_Stack_Commons.column_names import filename_dict, valid_table_names0 # , bin_names0, remove_from_list +from .column_names import filename_dict, valid_table_names0 # , bin_names0, remove_from_list def make_validation_table(fitspath): @@ -87,6 +87,7 @@ def make_validation_table(fitspath): print(" ") ''' + def compare_to_by_eye(fitspath, dataset): """ Purpose -> This function takes the automated validation table and checks it against