# Model arguments
model_name_or_path: Meta-Llama/Meta-Llama-3.1-8B
tokenizer_name_or_path: Meta-Llama/Meta-Llama-3.1-8B-Instruct
model_revision: main
torch_dtype: bfloat16
attn_implementation: flash_attention_2
use_liger: true
bf16: true
tf32: true
output_dir: runs/llama-3-1-8b-math-orca-qlora-10k-ep1
 
# Dataset arguments
dataset_id_or_path: train_dataset.json
max_seq_length: 1024
packing: true
 
# LoRA arguments
use_peft: true
load_in_4bit: true
lora_target_modules: "all-linear"
# important as we need to train the special tokens for the chat template of llama 
lora_modules_to_save: ["lm_head", "embed_tokens"] # you might need to change this for qwen or other models
lora_r: 16
lora_alpha: 16
 
# Training arguments
num_train_epochs: 1
per_device_train_batch_size: 8
gradient_accumulation_steps: 2
gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false
learning_rate: 2.0e-4 
lr_scheduler_type: constant
warmup_ratio: 0.1
 
# Logging arguments
logging_strategy: steps
logging_steps: 5
report_to:
- tensorboard
save_strategy: "epoch"
seed: 42
 
# Hugging Face Hub 
push_to_hub: true
# hub_model_id: llama-3-1-8b-math-orca-qlora-10k-ep1 # if not defined same as output_dir
hub_strategy: every_save