i3-lab/i3-500m

config = {
    "d_model": 1180,         
    "n_rwkv_layers": 16,  
    "n_attn_layers": 4,    
    "n_heads": 10,
    "seq_len": 1024,        
    "batch_size": 4,
    "accum_steps": 8,      
    "lr": 4e-4,
}