sec5b_ml_model_binaryclass_pipeline.py

import sys
import warnings
import numpy as np
import pandas as pd
import pickle
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import make_pipeline, Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
from sklearn.metrics import classification_report
import sec1_data_preparation as data_prep
import sec2_prepped_data_import as prepped_data_import
import sec3_feature_gen as feature_generate
import sec5_ml_model as ml_model

local_control_panel = {
	'done_switch': False,
}

# Main class
######################################################################
class MachineLearningNameEthnicityProjectBinaryClass(ml_model.MachineLearning, 
	data_prep.DataPreparationNameEthnicityProject):
	def __init__(self, control_panel):
		assert control_panel is not None, 'Error: `control_panel` is not declared.'
		assert None not in [control_panel['save_result_switch'], control_panel['use_subsampled_df_switch'],
							control_panel['use_subsampled_df_nk'], control_panel['use_featured_df_switch'],
							control_panel['use_feature_set'], control_panel['feature_selection_switch'],
							control_panel['random_state'], control_panel['cross_validation_switch'], 
							control_panel['ml_process_on_test_data_switch'], control_panel['ml_process_on_training_data_switch'], 
							control_panel['ml_process_on_ext_data_switch'],	control_panel['binary_target_label']
							], 'Error: `control_panel` is not declared properly.'
		'''Only one True can be in `cross_validation_switch`, `feature_selection_switch`, and 
		`ml_process_on_test_data_switch`/`ml_process_on_training_data_switch`/`ml_process_on_ext_data_switch`. 
		While `ml_process_on_test_data_switch`/`ml_process_on_training_data_switch`/`ml_process_on_ext_data_switch` can be 
		all/any True'''
		true_count_in_ml_process = sum([control_panel['ml_process_on_test_data_switch'], 
			control_panel['ml_process_on_training_data_switch'], control_panel['ml_process_on_ext_data_switch']])
		if true_count_in_ml_process >= 1:
			assert (sum([bool(x) for x in [control_panel['cross_validation_switch'], control_panel['feature_selection_switch'], 
				control_panel['ml_process_on_test_data_switch'], control_panel['ml_process_on_training_data_switch'], 
				control_panel['ml_process_on_ext_data_switch']]]) == true_count_in_ml_process), \
				('Total number of True value is incorrect amongst: `cross_validation_switch`, `feature_selection_switch`, ' 
				 '`ml_process_on_test_data_switch`, `ml_process_on_training_data_switch`, `ml_process_on_ext_data_switch`.')
		else:
			assert (sum([bool(x) for x in [control_panel['cross_validation_switch'], control_panel['feature_selection_switch'], 
				control_panel['ml_process_on_test_data_switch'], control_panel['ml_process_on_training_data_switch'], 
				control_panel['ml_process_on_ext_data_switch']]]) == 1), \
				('Total number of True value is incorrect amongst: `cross_validation_switch`, `feature_selection_switch`, ' 
				 '`ml_process_on_test_data_switch`, `ml_process_on_training_data_switch`, `ml_process_on_ext_data_switch`.')
		assert control_panel['eval_score'] in ['accuracy', 'balanced accuracy', 'macro f1 score', 'macro precision', 
			'macro recall', 'macro roc auc', None], 'Error: `control_panel["eval_score"]` is not declared properly.'
		super().__init__()
		super().dir_name()
		self.control_panel = control_panel
		# Conditionally re-assign
		if self.control_panel['use_subsampled_df_switch'] in [False, None]:
			self.control_panel['use_subsampled_df_nk'] = []
		if self.control_panel['use_featured_df_switch'] in [False, None]:
			self.control_panel['use_feature_set'] = []
		# Reassigning
		self.control_panel['binary_target_label'] = self.control_panel['binary_target_label'].lower()	
		# Decision boundary threshold to classify binary ethnicity target label
		self.d_threshold = 0.5

	def import_processed_main_data(self):
		file_full_data_with_features = 'Prepped_CanadianCensus1901_FeatureGenerated.csv'
		file_full_data_without_features = 'Prepped_CanadianCensus1901_NoFeatureGenerated.csv'
		file_subsampled_data_with_features = 'Prepped_CanadianCensus1901_FeatureGenerated_Seed888_N{}K.csv'.format(
			self.control_panel['use_subsampled_df_nk'])
		file_subsampled_data_without_features = 'Prepped_CanadianCensus1901_NoFeatureGenerated_Seed888_N{}K.csv'.format(
			self.control_panel['use_subsampled_df_nk'])
		t_obj = prepped_data_import.PreppedDataImportNameEthnicityProject()
		t_obj.control_panel['df_subsampling_switch'] = False

		if self.control_panel['use_subsampled_df_switch']:
			if self.control_panel['use_featured_df_switch']:
				t_obj.control_panel['filename'] = file_subsampled_data_with_features
				t_obj.prepped_data_import_steps()
				self.df_census1901_withFeat = t_obj.return_df()

			else:
				t_obj.control_panel['filename'] = file_subsampled_data_without_features
				t_obj.prepped_data_import_steps()
				self.df_census1901_noFeat = t_obj.return_df()		
			
		else:
			if self.control_panel['use_featured_df_switch']:
				t_obj.control_panel['filename'] = file_full_data_with_features
				t_obj.prepped_data_import_steps()
				self.df_census1901_withFeat = t_obj.return_df()

			else:
				t_obj.control_panel['filename'] = file_full_data_without_features
				t_obj.prepped_data_import_steps()
				self.df_census1901_noFeat = t_obj.return_df()

	def data_prep(self):
		var_tree = self.var_tree()
		# Declare which df to use. Can be `df_census1901_withFeat` or `df_census1901_noFeat`
		if self.control_panel['use_featured_df_switch']:
			self.df = self.df_census1901_withFeat.copy()
		else:
			self.df = self.df_census1901_noFeat.copy()

		# Declare feature set labels
		if self.control_panel['use_featured_df_switch']:
			self.feature_list_num, self.feature_list_cat, self.feature_list_liststring, self.feature_list_liststring1letter \
				= self.feature_use(self.control_panel['use_feature_set'])
		else:
			self.feature_list_num, self.feature_list_cat, self.feature_list_liststring, self.feature_list_liststring1letter \
				= [], var_tree['sex']['prepped'] + var_tree['name']['prepped'] + var_tree['loc']['prepped'], [], []
		self.feature_list_all = self.feature_list_num + self.feature_list_cat + self.feature_list_liststring + self.feature_list_liststring1letter

		# Declare predicted outcome label.
		label_tree = self.label_tree()
		if self.control_panel['binary_target_label'] in label_tree['ab_groups']:
			self.pred_outcome = var_tree['ethnic']['final_cat'][1]
		elif self.control_panel['binary_target_label'] in label_tree['fn_tribal_groups']:
			self.pred_outcome = var_tree['ethnic']['final_cat'][2]
		elif self.control_panel['binary_target_label'] in label_tree['fn_language_groups']:
			self.pred_outcome = var_tree['ethnic']['final_cat'][3]
		elif self.control_panel['binary_target_label'] in label_tree['main_groups']: 
			self.pred_outcome = var_tree['ethnic']['final_cat'][0]
		else:
			raise ValueError('Error: Incorrect value for control_panel["binary_target_label"].')

		# Binarize outcome in df
		self.pred_outcome_vFinal = self.pred_outcome+'_VFINAL'
		self.df[self.pred_outcome_vFinal] = np.where(self.df[self.pred_outcome]==self.control_panel['binary_target_label'], 
			self.control_panel['binary_target_label'], '~{}'.format(self.control_panel['binary_target_label']))
		assert len(self.df[self.df[self.pred_outcome_vFinal]==self.control_panel['binary_target_label']])>0, \
			'Error: Count of target label is 0.'
		# Data split
		self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(
			self.df[self.feature_list_all], self.df[self.pred_outcome_vFinal], 
			test_size=0.20, random_state=self.control_panel['random_state'])
		X_temp, self.X_dev, y_temp, self.y_dev = train_test_split(
			self.X_train, self.y_train, test_size=0.125, random_state=self.control_panel['random_state'])
		del X_temp, y_temp

	def feature_prep(self):
		transformer_num = Pipeline(steps=[
			('imputer', SimpleImputer(strategy='median')),
			('scaler', StandardScaler())])

		transformer_cat = Pipeline(steps=[
			('imputer', SimpleImputer(strategy='constant', fill_value='')),
			('onehotencoder', OneHotEncoder(handle_unknown='ignore'))])

		transformer_liststring_2orMoreLetters = Pipeline(steps=[
			('imputer', SimpleImputer(strategy='constant', fill_value='')),
			('ravel', RavelTransformer()),
			('countvectorizer', CountVectorizer(analyzer='word', tokenizer=None, preprocessor=None, 
				stop_words=None, max_features=5000))])

		transformer_liststring_1letter = Pipeline(steps=[
			('imputer', SimpleImputer(strategy='constant', fill_value='')),
			('ravel', RavelTransformer()),
			('countvectorizer', CountVectorizer(analyzer='char', tokenizer=None, preprocessor=None, 
				stop_words=None, max_features=5000))])

		liststring_dict = {
				'NAME_FIRST_2LETTER_SUBSTRINGS': [], 'NAME_FIRST_3LETTER_SUBSTRINGS': [], 'NAME_FIRST_4LETTER_SUBSTRINGS': [], 
				'NAME_FIRST_5LETTER_SUBSTRINGS': [], 'NAME_FIRST_6LETTER_SUBSTRINGS': [], 'NAME_MIDDLE_2LETTER_SUBSTRINGS': [], 
				'NAME_MIDDLE_3LETTER_SUBSTRINGS': [], 'NAME_MIDDLE_4LETTER_SUBSTRINGS': [], 'NAME_MIDDLE_5LETTER_SUBSTRINGS': [], 
				'NAME_MIDDLE_6LETTER_SUBSTRINGS': [], 'NAME_LAST_2LETTER_SUBSTRINGS': [], 'NAME_LAST_3LETTER_SUBSTRINGS': [], 
				'NAME_LAST_4LETTER_SUBSTRINGS': [], 'NAME_LAST_5LETTER_SUBSTRINGS': [], 'NAME_LAST_6LETTER_SUBSTRINGS': [], 
				'NAME_FIRST_1LETTER_SUBSTRINGS': [], 'NAME_MIDDLE_1LETTER_SUBSTRINGS': [], 'NAME_LAST_1LETTER_SUBSTRINGS': [], 
				'NAME_FULL_1LETTER_SUBSTRINGS': [], 'NAME_FULL_METAPHONE': [], 'LOC_ENTITY_LIST': [],
			}

		for key in liststring_dict:
			if key in self.feature_list_liststring:
				liststring_dict[key] = [key]

		self.preprocessor = ColumnTransformer(
			transformers=[
				('num', transformer_num, self.feature_list_num),
				('cat', transformer_cat, self.feature_list_cat),
				('liststring_firstName2Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FIRST_2LETTER_SUBSTRINGS']),
				('liststring_firstName3Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FIRST_3LETTER_SUBSTRINGS']),
				('liststring_firstName4Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FIRST_4LETTER_SUBSTRINGS']),
				('liststring_firstName5Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FIRST_5LETTER_SUBSTRINGS']),
				('liststring_firstName6Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FIRST_6LETTER_SUBSTRINGS']),
				('liststring_middleName2Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_MIDDLE_2LETTER_SUBSTRINGS']),
				('liststring_middleName3Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_MIDDLE_3LETTER_SUBSTRINGS']),
				('liststring_middleName4Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_MIDDLE_4LETTER_SUBSTRINGS']),
				('liststring_middleName5Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_MIDDLE_5LETTER_SUBSTRINGS']),
				('liststring_middleName6Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_MIDDLE_6LETTER_SUBSTRINGS']),
				('liststring_lastName2Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_LAST_2LETTER_SUBSTRINGS']),
				('liststring_lastName3Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_LAST_3LETTER_SUBSTRINGS']),
				('liststring_lastName4Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_LAST_4LETTER_SUBSTRINGS']),
				('liststring_lastName5Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_LAST_5LETTER_SUBSTRINGS']),
				('liststring_lastName6Lsubstrings', transformer_liststring_2orMoreLetters, liststring_dict['NAME_LAST_6LETTER_SUBSTRINGS']),
				('liststring_firstName1Lsubstrings', transformer_liststring_1letter, liststring_dict['NAME_FIRST_1LETTER_SUBSTRINGS']),
				('liststring_middleName1Lsubstrings', transformer_liststring_1letter, liststring_dict['NAME_MIDDLE_1LETTER_SUBSTRINGS']),
				('liststring_lastName1Lsubstrings', transformer_liststring_1letter, liststring_dict['NAME_LAST_1LETTER_SUBSTRINGS']),
				('liststring_fullName1Lsubstrings', transformer_liststring_1letter, liststring_dict['NAME_FULL_1LETTER_SUBSTRINGS']),
				('liststring_metaphones', transformer_liststring_2orMoreLetters, liststring_dict['NAME_FULL_METAPHONE']),
				('liststring_locSubstrings', transformer_liststring_2orMoreLetters, liststring_dict['LOC_ENTITY_LIST']),
				])

	def feature_selection(self):
		'''To use dev dataset to help select/confirm the predictability of features/feature sets.'''
		if self.control_panel['feature_selection_switch']:
			self.score = self.eval_score(self.control_panel['eval_score'])
			pipe = make_pipeline(self.preprocessor, self.control_panel['ml_algo_param_grid'][1]['clf'])
			self.grid = GridSearchCV(pipe, self.control_panel['ml_algo_param_grid'][1]['param'], 
				n_jobs=-1, cv=5, scoring=self.score)
			self.grid.fit(self.X_dev, self.y_dev)

	def feature_selection_result(self):
		if self.control_panel['feature_selection_switch']:
			print('>> Setting:\n', self.control_panel)
			print('>> Model parameter:')
			print(self.grid.best_estimator_)
			print('>> Best selected parameter:')
			print(self.grid.best_params_)
			print('>> Model score ({}): %.3f'.format(self.control_panel['eval_score']) % self.grid.best_score_)
			print('//////////////////////\n'*2)

			if self.control_panel['save_result_switch']:
				with open('{}{}{}_{}_SubsampleIs{}_ScoreIs{}_EthIs{}.txt'.format(self.result_dir, 'BinaryClass_FeatureSelection_', 
					self.control_panel['ml_algo_param_grid'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['eval_score'].capitalize().replace(
						' ', ''), self.control_panel['binary_target_label'].capitalize()), 'w') as f:
					print('>> Setting:\n', self.control_panel, file=f)
					print('>> Model parameter:', file=f)
					print(self.grid.best_estimator_, file=f)
					print('>> Best selected parameter:', file=f)
					print(self.grid.best_params_, file=f)
					print('>> Model score ({}): %.3f'.format(self.control_panel['eval_score']) % self.grid.best_score_, file=f)
					print('//////////////////////\n'*2, file=f)

	def cross_validation(self):
		if self.control_panel['cross_validation_switch']:
			self.score = self.eval_score(self.control_panel['eval_score'])
			pipe = make_pipeline(self.preprocessor, self.control_panel['ml_algo_param_grid'][1]['clf'])
			self.grid = RandomizedSearchCV(pipe, param_distributions=self.control_panel['ml_algo_param_grid'][1]['param'], 
				n_jobs=-1, cv=5, scoring=self.score)
			self.grid.fit(self.X_dev, self.y_dev)

	def cross_validation_result(self):
		if self.control_panel['cross_validation_switch']:
			print('>> Setting:\n', self.control_panel)
			print('>> Best parameter:')
			print(self.grid.best_estimator_)
			print('>> Best selected parameter:')
			print(self.grid.best_params_)
			print('>> Model score ({}) in cross validation: %.3f'.format(self.control_panel['eval_score']) % 
				self.grid.best_score_)
			'''print('>> Model score ({}) in dev set: %.3f'.format(self.control_panel['eval_score']) % 
				self.grid.score(self.X_dev, self.y_dev))
			print('>> Model score ({}) in training set: %.3f'.format(self.control_panel['eval_score']) % 
				self.grid.score(self.X_train, self.y_train))
			print('>> Model score ({}) in test set: %.3f'.format(self.control_panel['eval_score']) % 
				self.grid.score(self.X_test, self.y_test))'''
			print('//////////////////////\n'*2)

			if self.control_panel['save_result_switch']:
				assert type(self.control_panel['cross_validation_repeat']) == int, 'Error: `control_panel` is not declared properly.'

				with open('{}{}{}_{}_SubsampleIs{}_ScoreIs{}_EthIs{}_Take{}.txt'.format(self.result_dir, 'BinaryClass_CV_BestParameters_', 
					self.control_panel['ml_algo_param_grid'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['eval_score'].capitalize().replace(' ', ''), 
					self.control_panel['binary_target_label'].capitalize(),	self.control_panel['cross_validation_repeat']), 'w') as f:
					print('>> Setting:\n', self.control_panel, file=f)
					print('>> Best parameter:', file=f)
					print(self.grid.best_estimator_, file=f)
					print('>> Best selected parameter:', file=f)
					print(self.grid.best_params_, file=f)
					print('>> Model score ({}) in cross validation: %.3f'.format(self.control_panel['eval_score']) % 
						self.grid.best_score_, file=f)
					'''print('>> Model score ({}) in dev set: %.3f'.format(self.control_panel['eval_score']) % 
						self.grid.score(self.X_dev, self.y_dev), file=f)
					print('>> Model score ({}) in training set: %.3f'.format(self.control_panel['eval_score']) % 
						self.grid.score(self.X_train, self.y_train), file=f)
					print('>> Model score ({}) in test set: %.3f'.format(self.control_panel['eval_score']) % 
						self.grid.score(self.X_test, self.y_test), file=f)'''
					print('//////////////////////\n'*2, file=f)

	def ml_process(self):
		'''To use training set to train via selected ML algos and feature sets, and evaluate on test set.'''
		if (self.control_panel['ml_process_on_test_data_switch']|self.control_panel['ml_process_on_ext_data_switch']):
			self.pipe = make_pipeline(self.preprocessor, self.control_panel['ml_algo'][1]['clf'])
			self.pipe.fit(self.X_train, self.y_train)

			if self.control_panel['save_result_switch']:
				fname = '{}{}{}_{}_SubsampleIs{}_EthIs{}.pickle'.format(self.result_dir, 'BinaryClass_', 
					self.control_panel['ml_algo'][0], self.control_panel['use_feature_set'], 
					str(self.control_panel['use_subsampled_df_switch']), 
					self.control_panel['binary_target_label'].capitalize())
				pickle.dump(self.pipe, open(fname, 'wb'))

	def ml_process_result_on_test_data(self, _save_df_with_predicted_labels=True):
		if self.control_panel['ml_process_on_test_data_switch']:
			if self.control_panel['use_subsampled_df_switch']:
				print('WARNING: Full data is not being used currently.\n')
			target_labels = [self.control_panel['binary_target_label'], 
				'~{}'.format(self.control_panel['binary_target_label'])]
			predicted_labels_bool = (self.pipe.predict_proba(self.X_test)[:, 0] > self.d_threshold).astype(bool)
			predicted_labels = np.vectorize(bool_to_str)(predicted_labels_bool, target_labels[0], target_labels[1]).astype(str)
			#predicted_labels = self.pipe.predict(self.X_test) # using default decision threshold as > 0.5
			predicted_prob, predicted_prob_inverted = self.pipe.predict_proba(self.X_test)[:, 1], \
				self.pipe.predict_proba(self.X_test)[:, 0]

			print('>>>> On test data <<<<')
			print('>> Setting:\n', self.control_panel, '\n')
			print('>> Class label count, in test set:\n', self.y_test.value_counts(), '\n')
			print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n')
			print('>> Classification report:\n', classification_report(self.y_test, predicted_labels, target_names=target_labels), '\n')
			print('>> Custom classification report:\n', self.custom_classification_report(self.y_test, predicted_labels), '\n')
			print('>> Custom scores:\n', self.auc_roc(self.y_test, predicted_prob), '\n', 
				self.avg_precision(self.y_test, predicted_prob_inverted, target_labels[0]), '\n')
			print('//////////////////////\n'*2)

			if self.control_panel['save_result_switch']:
				with open('{}{}{}_{}_SubsampleIs{}_EthIs{}.txt'.format(self.result_dir, 'BinaryClass_ML_Process_OnTestData_', 
					self.control_panel['ml_algo'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['binary_target_label'].capitalize()
					), 'w') as f:
					print('>>>> On test data <<<<', file=f)
					print('>> Setting:\n', self.control_panel, '\n', file=f)
					print('>> Class label count, in test set:\n', self.y_test.value_counts(), '\n', file=f)
					print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n', file=f)
					print('>> Classification report:\n', classification_report(self.y_test, predicted_labels, 
						target_names=target_labels), file=f)
					print('>> Custom classification report:\n', self.custom_classification_report(self.y_test, predicted_labels), 
						'\n', file=f)
					print('>> Custom scores:\n', self.auc_roc(self.y_test, predicted_prob), '\n', 
						self.avg_precision(self.y_test, predicted_prob_inverted, target_labels[0]), '\n', file=f)
					print('//////////////////////\n'*2, file=f)

			if _save_df_with_predicted_labels:
				df_y_test = pd.DataFrame(self.y_test)
				df_y_test['ETHNICITY_PRED'] = predicted_labels
				df_X_test = self.X_test.copy()
				df_full_test = df_y_test.merge(df_X_test, left_index=True, right_index=True)
				df_full_test.to_csv('{}{}{}_{}_SubsampleIs{}_EthIs{}.csv'.format(self.result_dir, 
					'BinaryClass_ML_Process_DFwPredLabels_OnTestData_', self.control_panel['ml_algo'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['binary_target_label'].capitalize()
					), sep=',', index=False)

	def ml_process_result_on_training_data(self):
		if self.control_panel['ml_process_on_training_data_switch']:
			if self.control_panel['use_subsampled_df_switch']:
				print('WARNING: Full data is not being used currently.\n')
			target_labels = [self.control_panel['binary_target_label'], 
				'~{}'.format(self.control_panel['binary_target_label'])]
			predicted_labels_bool = (self.pipe.predict_proba(self.X_train)[:, 0] > self.d_threshold).astype(bool)
			predicted_labels = np.vectorize(bool_to_str)(predicted_labels_bool, target_labels[0], target_labels[1]).astype(str)
			#predicted_labels = self.pipe.predict(self.X_train) # using default decision threshold as > 0.5
			predicted_prob, predicted_prob_inverted = self.pipe.predict_proba(self.X_train)[:, 1], \
				self.pipe.predict_proba(self.X_train)[:, 0]

			print('>>>> On training data <<<<')
			print('>> Setting:\n', self.control_panel, '\n')
			print('>> Class label count, in training set:\n', self.y_train.value_counts(), '\n')
			print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n')
			print('>> Classification report:\n', classification_report(self.y_train, predicted_labels, target_names=target_labels), '\n')
			print('>> Custom classification report:\n', self.custom_classification_report(self.y_train, predicted_labels), '\n')
			print('>> Custom scores:\n', self.auc_roc(self.y_train, predicted_prob), '\n', 
				self.avg_precision(self.y_train, predicted_prob_inverted, target_labels[0]), '\n')
			print('//////////////////////\n'*2)

			if self.control_panel['save_result_switch']:
				with open('{}{}{}_{}_SubsampleIs{}_EthIs{}.txt'.format(self.result_dir, 'BinaryClass_ML_Process_OnTrainingData_', 
					self.control_panel['ml_algo'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['binary_target_label'].capitalize()
					), 'w') as f:
					print('>>>> On training data <<<<', file=f)
					print('>> Setting:\n', self.control_panel, '\n', file=f)
					print('>> Class label count, in training set:\n', self.y_train.value_counts(), '\n', file=f)
					print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n', file=f)
					print('>> Classification report:\n', classification_report(self.y_train, predicted_labels, 
						target_names=target_labels), '\n', file=f)
					print('>> Custom classification report:\n', self.custom_classification_report(self.y_train, 
						predicted_labels), '\n', file=f)
					print('>> Custom scores:\n', self.auc_roc(self.y_train, predicted_prob), '\n', 
						self.avg_precision(self.y_train, predicted_prob_inverted, target_labels[0]), '\n', file=f)
					print('//////////////////////\n'*2, file=f)

	def ml_process_result_on_external_data(self):
		if self.control_panel['ml_process_on_ext_data_switch']:
			# Transform external data with the same feature generation steps
			feature_tranform_obj = feature_transform_external_data()
			feature_tranform_obj.import_processed_main_data()
			feature_tranform_obj.data_prep()
			feature_tranform_obj.create_features(on_switch=True)
			self.df_external_processed = feature_tranform_obj.df_main
			# Prepare ethnicity outcome variable
			self.df_external_processed[self.pred_outcome_vFinal] = np.where(
				self.df_external_processed['ETHNICITY_RECAT'].str.lower()==self.control_panel['binary_target_label'], 
				self.control_panel['binary_target_label'], '~{}'.format(self.control_panel['binary_target_label']))
			# Data split
			self.X_ext_test = self.df_external_processed[self.feature_list_all]
			self.y_ext_test = self.df_external_processed[self.pred_outcome_vFinal]
			# Apply trained model and predict in external data
			target_labels = [self.control_panel['binary_target_label'], 
				'~{}'.format(self.control_panel['binary_target_label'])]		
			predicted_labels_bool = (self.pipe.predict_proba(self.X_ext_test)[:, 0] > self.d_threshold).astype(bool)
			predicted_labels = np.vectorize(bool_to_str)(predicted_labels_bool, target_labels[0], target_labels[1]).astype(str)
			#predicted_labels = self.pipe.predict(self.X_ext_test) # using default decision threshold as > 0.5
			predicted_prob, predicted_prob_inverted = self.pipe.predict_proba(self.X_ext_test)[:, 1], \
				self.pipe.predict_proba(self.X_ext_test)[:, 0]

			print('>>>> On external data <<<<')
			print('>> Setting:\n', self.control_panel, '\n')
			print('>> Class label count, in external test set:\n', self.y_ext_test.value_counts(), '\n')
			print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n')
			print('>> Classification report:\n', classification_report(self.y_ext_test, predicted_labels, 
				target_names=target_labels))
			print('>> Custom classification report:\n', self.custom_classification_report(self.y_ext_test, 
				predicted_labels), '\n')
			print('>> Custom scores:\n', self.auc_roc(self.y_ext_test, predicted_prob), '\n', 
				self.avg_precision(self.y_ext_test, predicted_prob_inverted, target_labels[0]), '\n')
			print('//////////////////////\n'*2)

			if self.control_panel['save_result_switch']:
				with open('{}{}{}_{}_SubsampleIs{}_EthIs{}.txt'.format(self.result_dir, 'BinaryClass_ML_Process_OnExtData_', 
					self.control_panel['ml_algo'][0], self.control_panel['use_feature_set'],
					str(self.control_panel['use_subsampled_df_switch']), self.control_panel['binary_target_label'].capitalize()
					), 'w') as f:
					print('>>>> On external data <<<<', file=f)
					print('>> Setting:\n', self.control_panel, '\n', file=f)
					print('>> Class label count, in external test set:\n', self.y_ext_test.value_counts(), '\n', file=f)
					print('>> Class label count, predicted:\n', pd.Series(predicted_labels).value_counts(), '\n', file=f)
					print('>> Classification report:\n', classification_report(self.y_ext_test, predicted_labels, 
						target_names=target_labels), file=f)
					print('>> Custom classification report:\n', self.custom_classification_report(self.y_ext_test, 
						predicted_labels), '\n', file=f)
					print('>> Custom scores:\n', self.auc_roc(self.y_ext_test, predicted_prob), '\n', 
						self.avg_precision(self.y_ext_test, predicted_prob_inverted, target_labels[0]), '\n', file=f)
					print('//////////////////////\n'*2, file=f)

# Helper class and function
######################################################################
class RavelTransformer(BaseEstimator, TransformerMixin):
	def fit(self, X, y=None):
		return self

	def transform(self, X, y=None):
		return X.ravel()

class feature_transform_external_data(feature_generate.FeatureCreationNameEthnicityProject, ml_model.ExternalData):
	def import_processed_main_data(self):
		mock_external_data = self.create_mock_external_data()
		self.df_main = mock_external_data.copy()

def bool_to_str(val, true_as, false_as):
	if val == True:
		return true_as
	elif val == False:
		return false_as
	else: 
		raise ValidationError(f'{val} is not valid value.')