codefinetuner/config/codefinetuner_config.yaml at master · cuolm/codefinetuner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# shared across all stages
globals: &globals
  workspace_path: null  # null: defaults to current working directory (cwd)
  model_name: "unsloth/qwen2.5-coder-0.5b"
  fim_prefix_token: "<|fim_prefix|>"
  fim_middle_token: "<|fim_middle|>"
  fim_suffix_token: "<|fim_suffix|>"
  fim_pad_token: "<|fim_pad|>"
  eos_token: "<|endoftext|>"
  label_pad_token_id: -100
  max_token_sequence_length: 1024
  data_language: "c"
  data_extensions: [".c", ".h"]
  use_unsloth: true

# preprocess stage
preprocess:
  <<: *globals
  split_mode: "manual"
  train_ratio: 0.8  # ignored when split mode manual
  eval_ratio: 0.1  # ignored when split mode manual
  test_ratio: 0.1  # ignored when split mode manual
  max_code_blocks_ast_depth: 8
  min_middle_tokens_length: 8
  max_middle_tokens_length: 256
  fim_examples_per_subblock_ratio: 0.2
  rand_to_ast_fim_examples_ratio: 0.0
  rand_examples_min_prefix_suffix_tokens_length: 8
  tokenizer_batch_size: 32
  raw_data_path: null  # null: defaults to <workspace_path>/data
  tree_sitter_parser_path: null  # uses tree_sitter_language_pack parser
  tree_sitter_definitions_path: null  # null: uses package internal tree sitter definitions
  rng_seed: 0

# finetune stage
finetune:
  <<: *globals
  model_attn_implementation: "sdpa"
  lora_r: 32
  lora_alpha: 64
  lora_dropout: 0.0
  lora_bias: "none"
  lora_target_modules: ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "down_proj", "up_proj"]
  selected_checkpoint_strategy : "best"  # "best" or "last"
  trainer_resume_from_checkpoint: null  # null->fresh start, "last"->take last checkpoint
  trainer_clear_checkpoint_dir: false
  trainer_num_train_epochs: 1
  trainer_per_device_train_batch_size: 1
  trainer_per_device_eval_batch_size: 1
  trainer_gradient_accumulation_steps: 8
  trainer_learning_rate: 5e-5
  trainer_weight_decay: 0.01
  trainer_max_grad_norm: 1.0
  trainer_lr_scheduler_type: "cosine"
  trainer_warmup_steps: 20
  trainer_gradient_checkpointing: true
  trainer_logging_steps: 10
  trainer_eval_strategy: "steps"
  trainer_eval_steps: 20
  trainer_save_strategy: "steps"
  trainer_save_steps: 20
  trainer_logging_strategy: "steps"
  dataset_shuffle_buffer_size: 50000
  dataset_shuffle_seed: 0

# evaluate stage
evaluate:
  <<: *globals
  benchmark_sample_size: 100
  benchmark_shuffle_buffer_size: 10000000
  benchmark_shuffle_seed: 42
  generation_checkpoint: "pipeline"  # "pipeline" -> selected checkpoint from finetune stage, "checkpoint-name" -> specific checkpoint
  generation_batch_size: 5
  generation_max_new_tokens: 128
  generation_do_sample: false
  generation_temperature: 0.1  # only active if generation_do_sample: true
  generation_top_p: 0.99  # only active if generation_do_sample: true
  codebleu_ngram_weight: 0.1
  codebleu_weighted_ngram_weight: 0.1
  codebleu_syntax_ast_weight: 0.4
  codebleu_dataflow_weight: 0.4
  sentencebleu_ngram_weight_1: 0.25
  sentencebleu_ngram_weight_2: 0.25
  sentencebleu_ngram_weight_3: 0.25
  sentencebleu_ngram_weight_4: 0.25
  line_match_number_of_lines: 2
  plot_only: false
  benchmark_use_existing_dataset: false

# convert stage
convert:
  <<: *globals

# mlflow tracking (optional)
tracking:
  <<: *globals
  use_mlflow: false
  mlflow_tracking_path: null   # null -> <workspace_path>/outputs/mlflow
  mlflow_experiment_name: "codefinetuner"
  mlflow_model_logging_strategy: "none"  # "none", "adapter", "gguf", or "all"