From 62c2600702547a7a508a450869a0d6a05102ab85 Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 16:08:11 +0200 Subject: [PATCH 1/6] this test never close the process after exporting the model --- bin/run-ci-mailabs_time.sh | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 bin/run-ci-mailabs_time.sh diff --git a/bin/run-ci-mailabs_time.sh b/bin/run-ci-mailabs_time.sh new file mode 100755 index 000000000..2c3b2a859 --- /dev/null +++ b/bin/run-ci-mailabs_time.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +set -xe + +mailabs_dir=$1 #"./data/M-AILABS" +mailabs_lang=$2 #"fr_FR" +alphabet_path=$3 #"./data/fr_FR/alphabet.txt" +scorer_path=$4 #"./data/fr_FR/fr_lm.scorer" +mailabs_train_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_train.csv" +mailabs_dev_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_test.csv" +mailabs_test_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_test.csv" + + +epoch_count=1 +audio_sample_rate=16000 + +if [ ! -f "${mailabs_train_csv}" ]; then + echo "Downloading and preprocessing M-AILABS data, saving in ${mailabs_dir}${mailabs_lang}/." + python -u bin/import_m-ailabs.py ${mailabs_dir} --language ${mailabs_lang} +fi; + +st=`date +%s` +echo "Index 0 starts at ${st}." + +python -u train.py --alphabet_config_path ${alphabet_path} \ + --show_progressbar false --early_stop false \ + --train_files ${mailabs_train_csv} --train_batch_size 32 \ + --feature_cache '/tmp/mailabs_cache' \ + --dev_files ${ldc93s1_csv} --dev_batch_size 32 \ + --test_files ${ldc93s1_csv} --test_batch_size 32 \ + --n_hidden 100 --epochs $epoch_count \ + --max_to_keep 1 --checkpoint_dir '/tmp/mailabs_ckpt' \ + --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/mailabs_train' \ + --scorer_path ${scorer_path} \ + --audio_sample_rate ${audio_sample_rate} \ + --export_tflite false + --automatic_mixed_precision 1 + --train_cudnn 1 + +exit_code=$? + +ent=`date +%s` +echo "Index -1 ends at ${ent}" + +ext=`expr $end_time - $start_time` +echo "Execution took ${ext} seconds to return exit code ${exit_code}." \ No newline at end of file From 9143424b76e15ac70022a2560798f2e03ebf12ac Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 16:38:48 +0200 Subject: [PATCH 2/6] added log level debug for training --- bin/run-ci-mailabs_time.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run-ci-mailabs_time.sh b/bin/run-ci-mailabs_time.sh index 2c3b2a859..a57988ae4 100755 --- a/bin/run-ci-mailabs_time.sh +++ b/bin/run-ci-mailabs_time.sh @@ -36,6 +36,7 @@ python -u train.py --alphabet_config_path ${alphabet_path} \ --export_tflite false --automatic_mixed_precision 1 --train_cudnn 1 + --log_level 0 exit_code=$? From 878be452ee9774efd9c6546183f9200f65d6ff52 Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 16:46:41 +0200 Subject: [PATCH 3/6] code format --- bin/run-ci-mailabs_time.sh | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/bin/run-ci-mailabs_time.sh b/bin/run-ci-mailabs_time.sh index a57988ae4..85297252e 100755 --- a/bin/run-ci-mailabs_time.sh +++ b/bin/run-ci-mailabs_time.sh @@ -2,24 +2,23 @@ set -xe -mailabs_dir=$1 #"./data/M-AILABS" -mailabs_lang=$2 #"fr_FR" +mailabs_dir=$1 #"./data/M-AILABS" +mailabs_lang=$2 #"fr_FR" alphabet_path=$3 #"./data/fr_FR/alphabet.txt" -scorer_path=$4 #"./data/fr_FR/fr_lm.scorer" +scorer_path=$4 #"./data/fr_FR/fr_lm.scorer" mailabs_train_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_train.csv" mailabs_dev_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_test.csv" mailabs_test_csv="${mailabs_dir}${mailabs_lang}/${mailabs_lang}_test.csv" - epoch_count=1 audio_sample_rate=16000 if [ ! -f "${mailabs_train_csv}" ]; then - echo "Downloading and preprocessing M-AILABS data, saving in ${mailabs_dir}${mailabs_lang}/." - python -u bin/import_m-ailabs.py ${mailabs_dir} --language ${mailabs_lang} -fi; + echo "Downloading and preprocessing M-AILABS data, saving in ${mailabs_dir}${mailabs_lang}/." + python -u bin/import_m-ailabs.py ${mailabs_dir} --language ${mailabs_lang} +fi -st=`date +%s` +st=$(date +%s) echo "Index 0 starts at ${st}." python -u train.py --alphabet_config_path ${alphabet_path} \ @@ -30,18 +29,18 @@ python -u train.py --alphabet_config_path ${alphabet_path} \ --test_files ${ldc93s1_csv} --test_batch_size 32 \ --n_hidden 100 --epochs $epoch_count \ --max_to_keep 1 --checkpoint_dir '/tmp/mailabs_ckpt' \ - --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/mailabs_train' \ + --learning_rate 0.001 --dropout_rate 0.05 --export_dir '/tmp/mailabs_train' \ --scorer_path ${scorer_path} \ --audio_sample_rate ${audio_sample_rate} \ - --export_tflite false - --automatic_mixed_precision 1 - --train_cudnn 1 + --export_tflite false \ + --automatic_mixed_precision 1 \ + --train_cudnn 1 \ --log_level 0 exit_code=$? -ent=`date +%s` +ent=$(date +%s) echo "Index -1 ends at ${ent}" -ext=`expr $end_time - $start_time` -echo "Execution took ${ext} seconds to return exit code ${exit_code}." \ No newline at end of file +ext=$(expr $end_time - $start_time) +echo "Execution took ${ext} seconds to return exit code ${exit_code}." From 6459b7b7f583ab31c76a20a355f2d851849b1998 Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 17:18:33 +0200 Subject: [PATCH 4/6] fix timer --- bin/run-ci-mailabs_time.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-ci-mailabs_time.sh b/bin/run-ci-mailabs_time.sh index 85297252e..94d8568e2 100755 --- a/bin/run-ci-mailabs_time.sh +++ b/bin/run-ci-mailabs_time.sh @@ -42,5 +42,5 @@ exit_code=$? ent=$(date +%s) echo "Index -1 ends at ${ent}" -ext=$(expr $end_time - $start_time) +ext=$(expr $ent - $st) echo "Execution took ${ext} seconds to return exit code ${exit_code}." From 27124d274ea06d6e0598b9766f806089f3e49fec Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 17:29:49 +0200 Subject: [PATCH 5/6] fix unkown flags --- bin/run-ci-mailabs_time.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/run-ci-mailabs_time.sh b/bin/run-ci-mailabs_time.sh index 94d8568e2..785346814 100755 --- a/bin/run-ci-mailabs_time.sh +++ b/bin/run-ci-mailabs_time.sh @@ -33,8 +33,6 @@ python -u train.py --alphabet_config_path ${alphabet_path} \ --scorer_path ${scorer_path} \ --audio_sample_rate ${audio_sample_rate} \ --export_tflite false \ - --automatic_mixed_precision 1 \ - --train_cudnn 1 \ --log_level 0 exit_code=$? From 9b8a8e4e0fcc4e8efedbf44c04c7d2272b82c258 Mon Sep 17 00:00:00 2001 From: wasertech Date: Fri, 22 Apr 2022 22:33:13 +0200 Subject: [PATCH 6/6] no memory test fix hanging --- training/coqui_stt_training/train.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/training/coqui_stt_training/train.py b/training/coqui_stt_training/train.py index 7bff29ac1..9d8d172bc 100644 --- a/training/coqui_stt_training/train.py +++ b/training/coqui_stt_training/train.py @@ -324,15 +324,15 @@ def create_training_datasets( def train(): - log_info("Performing dummy training to check for memory problems.") - log_info( - "If the following process crashes, you likely have batch sizes " - "that are too big for your available system memory (or GPU memory)." - ) - train_impl(epochs=1, reverse=True, limit=Config.train_batch_size * 3, write=False) - - log_info("Dummy run finished without problems, now starting real training process.") - train_impl(epochs=Config.epochs, silent_load=True) + #log_info("Performing dummy training to check for memory problems.") + #log_info( + # "If the following process crashes, you likely have batch sizes " + # "that are too big for your available system memory (or GPU memory)." + #) + #train_impl(epochs=1, reverse=True, limit=Config.train_batch_size * 3, write=False) + + #log_info("Dummy run finished without problems, now starting real training process.") + train_impl(epochs=Config.epochs) #, silent_load=True) def train_impl(epochs=0, reverse=False, limit=0, write=True, silent_load=False):