laguna-s-2.1:nvfp4

110.4K 9 minutes ago

Our most capable model to date, designed for long-horizon work. 70.2% on Terminal-Bench 2.1 at 118B-A8B.

tools thinking
... /
json
d575474e42ff · 8.2kB
{
"architectures": [
"LagunaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"auto_map": {
"AutoConfig": "configuration_laguna.LagunaConfig",
"AutoModelForCausalLM": "modeling_laguna.LagunaForCausalLM"
},
"bos_token_id": 2,
"decoder_sparse_step": 1,
"dtype": "bfloat16",
"eos_token_id": [
2,
24
],
"gating": "per-head",
"gating_types": [
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head",
"per_head"
],
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 3072,
"initializer_range": 0.02,
"intermediate_size": 12288,
"layer_types": [
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention",
"full_attention",
"sliding_attention",
"sliding_attention",
"sliding_attention"
],
"max_position_embeddings": 1048576,
"mlp_layer_types": [
"dense",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse",
"sparse"
],
"mlp_only_layers": [
0
],
"model_type": "laguna",
"moe_apply_router_weight_on_input": false,
"moe_intermediate_size": 1024,
"moe_routed_scaling_factor": 2.5,
"moe_router_logit_softcapping": 0.0,
"norm_topk_prob": true,
"num_attention_heads": 48,
"num_attention_heads_per_layer": [
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72,
48,
72,
72,
72
],
"num_experts": 256,
"num_experts_per_tok": 10,
"num_hidden_layers": 48,
"num_key_value_heads": 8,
"output_router_logits": false,
"pad_token_id": 9,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"full_attention": {
"attention_factor": 1.4852030263919618,
"beta_fast": 32.0,
"beta_slow": 1.0,
"factor": 128.0,
"original_max_position_embeddings": 8192,
"partial_rotary_factor": 0.5,
"rope_theta": 500000.0,
"rope_type": "yarn"
},
"sliding_attention": {
"partial_rotary_factor": 1.0,
"rope_theta": 10000.0,
"rope_type": "default"
}
},
"router_aux_loss_coef": 0.0,
"shared_expert_intermediate_size": 1024,
"sliding_window": 512,
"tie_word_embeddings": false,
"transformers_version": "5.12.1",
"use_cache": true,
"vocab_size": 100352,
"quantization_config": {
"config_groups": {
"group_0": {
"weights": {
"dynamic": false,
"num_bits": 4,
"type": "float",
"group_size": 16
},
"targets": [
"Linear"
]
}
},
"ignore": [
"model.layers.1.mlp.gate",
"model.layers.10.mlp.gate",
"model.layers.11.mlp.gate",
"model.layers.12.mlp.gate",
"model.layers.13.mlp.gate",
"model.layers.14.mlp.gate",
"model.layers.15.mlp.gate",
"model.layers.16.mlp.gate",
"model.layers.17.mlp.gate",
"model.layers.18.mlp.gate",
"model.layers.19.mlp.gate",
"model.layers.2.mlp.gate",
"model.layers.20.mlp.gate",
"model.layers.21.mlp.gate",
"model.layers.22.mlp.gate",
"model.layers.23.mlp.gate",
"model.layers.24.mlp.gate",
"model.layers.25.mlp.gate",
"model.layers.26.mlp.gate",
"model.layers.27.mlp.gate",
"model.layers.28.mlp.gate",
"model.layers.29.mlp.gate",
"model.layers.3.mlp.gate",
"model.layers.30.mlp.gate",
"model.layers.31.mlp.gate",
"model.layers.32.mlp.gate",
"model.layers.33.mlp.gate",
"model.layers.34.mlp.gate",
"model.layers.35.mlp.gate",
"model.layers.36.mlp.gate",
"model.layers.37.mlp.gate",
"model.layers.38.mlp.gate",
"model.layers.39.mlp.gate",
"model.layers.4.mlp.gate",
"model.layers.40.mlp.gate",
"model.layers.41.mlp.gate",
"model.layers.42.mlp.gate",
"model.layers.43.mlp.gate",
"model.layers.44.mlp.gate",
"model.layers.45.mlp.gate",
"model.layers.46.mlp.gate",
"model.layers.47.mlp.gate",
"model.layers.5.mlp.gate",
"model.layers.6.mlp.gate",
"model.layers.7.mlp.gate",
"model.layers.8.mlp.gate",
"model.layers.9.mlp.gate"
],
"quant_algo": "W4A16_NVFP4",
"producer": {
"name": "modelopt",
"version": "0.47.0.dev75+gfbcdc16c2.d20260818"
},
"quant_method": "modelopt"
}
}