| batch_size_training: '4' |
| batching_strategy: padding |
| checkpoint_type: StateDictType.SHARDED_STATE_DICT |
| context_length: '8192' |
| curriculum_learning: 'False' |
| curriculum_phases: '3' |
| dataset: '[''OpenCoderSFTStage2'']' |
| ddp_timeout: '36000' |
| debug: 'False' |
| decay_steps: None |
| dist_checkpoint_folder: fine-tuned |
| drop_last: 'True' |
| dynamic_batch_size: 'False' |
| enable_deepspeed: 'False' |
| enable_fsdp: 'True' |
| enable_memory_profiling: 'False' |
| enable_memory_trace: 'False' |
| enable_mixed_precision: 'True' |
| enable_tensorboard: 'True' |
| eta_min: 1e-05 |
| eval_epoch: '1' |
| eval_in_memory: 'False' |
| eval_steps: '1000' |
| evaluation_strategy: steps |
| flop_counter: 'False' |
| flop_counter_start: '3' |
| fp16: 'False' |
| freeze_layers: 'False' |
| from_peft_checkpoint: '' |
| fsdp_activation_checkpointing: 'True' |
| fsdp_cpu_offload: 'False' |
| fsdp_cpu_ram_efficient_loading: 'False' |
| gamma: '0.85' |
| gradient_accumulation_steps: '8' |
| gradient_checkpointing: 'True' |
| gradient_checkpointing_kwargs: '{''use_reentrant'': False}' |
| gradient_clipping: 'False' |
| gradient_clipping_threshold: '1.0' |
| handle_long_sequences: 'True' |
| hf_hub_metrics_cache_dir: /shared/public/data/controlllm/metrics/ |
| hsdp: 'True' |
| learning_rate: 5e-05 |
| load_best_model_at_end: 'False' |
| logging_steps: '500' |
| long_sequence_threshold: '16384' |
| low_cpu_fsdp: 'False' |
| lr: '0.0001' |
| lr_scheduler_per_iter: 'True' |
| max_eval_step: '500' |
| max_grad_norm: '1.0' |
| max_step: '0' |
| max_tokens_per_batch: '-1' |
| max_train_step: '-1' |
| memory_per_token: '-1' |
| mixed_precision: 'True' |
| model_name: PATH/to/Model |
| no_cuda: 'False' |
| num_epochs: '3' |
| num_freeze_layers: '1' |
| num_train_epochs: '20' |
| num_unfrozen_layers: '8' |
| num_workers_dataloader: '0' |
| one_gpu: 'False' |
| optimizer: AdamW |
| overwrite_output_dir: 'False' |
| peft_method: lora |
| per_device_eval_batch_size: '1' |
| per_device_train_batch_size: '12' |
| precompute_batches: None |
| pure_bf16: 'False' |
| quantization: 'False' |
| replica_group_size: '1' |
| resume_checkpoint_folder: None |
| resume_from_latest: 'True' |
| run_validation: 'True' |
| save_epoch: '1' |
| save_metrics: 'False' |
| save_model: 'True' |
| save_optimizer: 'False' |
| save_steps: '1000' |
| seed: '42' |
| sharding_group_size: '8' |
| sharding_strategy: ShardingStrategy.HYBRID_SHARD |
| step_size: '1' |
| tokenizer_name: None |
| trainer: native |
| unfrozen_strategy: interweave |
| use_fast_kernels: 'False' |
| use_fp16: 'False' |
| use_peft: 'False' |
| use_profiler: 'False' |
| use_wandb: 'False' |
| val_batch_size: '1' |
| warmup_steps: '1000' |
| weight_decay: '0.01' |
| weight_decay_ratio: '0.1' |
|
|
