add finetuner_no_trainer

examples/finetune.py

@@ -15,6 +15,7 @@
 """
 
 import sys
+import os
 
 from transformers import HfArgumentParser
 
@@ -32,6 +33,7 @@
 def main():
 	# Parses arguments
     pipeline_name = "finetuner"
+    # pipeline_name = "finetuner_no_trainer"
     PipelineArguments = AutoArguments.get_pipeline_args_class(pipeline_name)
 
     parser = HfArgumentParser((ModelArguments, DatasetArguments, PipelineArguments))

scripts/run_finetune_no_trainer.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Please run this script under ${project_id} in project directory of
+#   https://github.com/shizhediao/llm-ft
+#     COMMIT: d5fecf30ba8011067b10cf51fede53a5ab6574e4
+
+deepspeed_args="--master_port=11000"      # Default argument
+if [ $# -ge 1 ]; then
+  deepspeed_args="$1"
+fi
+
+exp_id=finetune
+project_dir=$(cd "$(dirname $0)"/..; pwd)
+output_dir=${project_dir}/output_models/${exp_id}
+log_dir=${project_dir}/log/${exp_id}
+
+dataset_path=${project_dir}/data/alpaca/train
+
+mkdir -p ${output_dir} ${log_dir}
+
+deepspeed ${deepspeed_args} \
+  examples/finetune.py \
+    --model_name_or_path gpt2 \
+    --dataset_path ${dataset_path} \
+    --output_dir ${output_dir} --overwrite_output_dir \
+    --num_train_epochs 0.01 \
+    --learning_rate 2e-5 \
+    --block_size 512 \
+    --per_device_train_batch_size 1 \
+    --deepspeed configs/ds_config_zero2.json \
+    --bf16 \
+    --run_name finetune \
+    --validation_split_percentage 0 \
+    --logging_steps 20 \
+    --do_train \
+    --ddp_timeout 72000 \
+    --save_steps 5000 \
+    --dataloader_num_workers 1 \
+    | tee ${log_dir}/train.log \
+    2> ${log_dir}/train.err

src/lmflow/args.py

@@ -510,6 +510,7 @@ class InferencerArguments:
 
 PIPELINE_ARGUMENT_MAPPING = {
     "finetuner": FinetunerArguments,
+    "finetuner_no_trainer": FinetunerArguments,
     "evaluator": EvaluatorArguments,
     "inferencer": InferencerArguments,
 }

src/lmflow/datasets/dataset.py

@@ -58,7 +58,7 @@ def __init__(self, data_args=None, backend: str="huggingface", *args, **kwargs):
         if backend == "huggingface":
             data_files = [
                 x.absolute().as_posix()
-                 for x in Path(self.dataset_path).glob("*.json")
+                for x in Path(self.dataset_path).glob("*.json")
             ]
 
             # Iterate through all the files and ensure they have the same data type

src/lmflow/models/hf_decoder_model.py

@@ -286,6 +286,7 @@ def tokenize_function(examples):
                     output = self.tokenizer(
                         examples[text_column_name],
                         truncation=True,
+                        return_token_type_ids=False if 'galactica' in self.model_args.model_name_or_path else True,
                     )
             # clm input could be much much longer than block_size
             if "Token indices sequence length is longer than the" in cl.out:

src/lmflow/pipeline/auto_pipeline.py

@@ -4,13 +4,15 @@
 """
 
 from lmflow.pipeline.finetuner import Finetuner
+from lmflow.pipeline.finetuner_no_trainer import Finetuner_no_trainer
 from lmflow.pipeline.evaluator import Evaluator
 from lmflow.pipeline.inferencer import Inferencer
 
 
 PIPELINE_MAPPING = {
     "finetuner": Finetuner,
     "evaluator": Evaluator,
+    "finetuner_no_trainer": Finetuner_no_trainer,
     "inferencer": Inferencer,
 }