使用lora微调时，同时训练了一些层的参数，合并验证报错

[ac-automata]

Reminder

• I have read the README and searched the existing issues.

Reproduction

config

{
"alpha_pattern": {},
"auto_mapping": null,
"base_model_name_or_path": "/root/data/Mixtral-HQQ/MixTAO-7Bx2-MoE-v8.1",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layers_pattern": null,
"layers_to_transform": null,
"loftq_config": {},
"lora_alpha": 16,
"lora_dropout": 0.1,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": [
"input_layernorm",
"norm",
"gate_proj"
],
"peft_type": "LORA",
"r": 16,
"rank_pattern": {},
"revision": null,
"target_modules": [
"q_proj",
"v_proj"
],
"task_type": "CAUSAL_LM",
"use_dora": false,
"use_rslora": true
}

报错

Traceback (most recent call last):
File "/opt/anaconda3/envs/train/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/opt/anaconda3/envs/train/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/root/data/Mixtral-HQQ/LLaMA-Factory/src/llmtuner/train/tuner.py", line 32, in run_exp
run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks)
File "/root/data/Mixtral-HQQ/LLaMA-Factory/src/llmtuner/train/sft/workflow.py", line 91, in run_sft
predict_results = trainer.predict(dataset, metric_key_prefix="predict", **gen_kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/trainer_seq2seq.py", line 230, in predict
return super().predict(test_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/trainer.py", line 3309, in predict
output = eval_loop(
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/trainer.py", line 3422, in evaluation_loop
loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)
File "/root/data/Mixtral-HQQ/LLaMA-Factory/src/llmtuner/train/sft/trainer.py", line 47, in prediction_step
loss, generated_tokens, _ = super().prediction_step( # ignore the returned labels (may be truncated)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/trainer_seq2seq.py", line 296, in prediction_step
generated_tokens = self.model.generate(**generation_inputs, **gen_kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/peft/peft_model.py", line 1148, in generate
outputs = self.base_model.generate(*args, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/generation/utils.py", line 1592, in generate
return self.sample(
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/generation/utils.py", line 2696, in sample
outputs = self(
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/accelerate/hooks.py", line 166, in new_forward
output = module._old_forward(*args, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/transformers/models/mixtral/modeling_mixtral.py", line 1374, in forward
logits = self.lm_head(hidden_states)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl
return forward_call(*input, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/accelerate/hooks.py", line 166, in new_forward
output = module._old_forward(*args, **kwargs)
File "/opt/anaconda3/envs/train/lib/python3.9/site-packages/torch/nn/modules/linear.py", line 114, in forward
return F.linear(input, self.weight, self.bias)
RuntimeError: expected scalar type Float but found BFloat16

Expected behavior

No response

System Info

No response

Others

No response

[codemayq]

需要检查一下 训练时使用的精度 和 合并时的精度是否保持一致，都是 bf16或者fp16，否则可能会有错误。

[ac-automata]

需要手动合并吗？我只在预测时加上 LoRA 权重，都使用的bf16

需要检查一下 训练时使用的精度 和 合并时的精度是否保持一致，都是 bf16或者fp16，否则可能会有错误。