codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 17, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 138412032, "shape": [ 6144 ], "size": 138412032, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 138436608, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 221011968, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 290217984, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 359424000, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 359448576, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 363773952, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 389726208, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 415678464, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 420839424, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 420864000, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 503439360, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 572645376, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 641851392, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 641875968, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 646201344, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 672153600, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 698105856, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 703266816, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 703291392, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 785866752, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 855072768, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 924278784, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 924303360, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 928628736, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 954580992, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 980533248, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 985694208, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 985718784, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1068294144, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 1137500160, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1206706176, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1206730752, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 1211056128, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1237008384, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.3.attn_v.weight", "offset": 1262960640, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1268121600, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1268146176, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1350721536, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 1419927552, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1489133568, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1489158144, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 1493483520, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1519435776, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 1545388032, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1550548992, "shape": [ 6144, 16384 ], "size": 5160960, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1619755008, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.5.attn_k.weight", "offset": 1688961024, "shape": [ 6144, 1024 ], "size": 69206016, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 1693286400, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1719238656, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 1745190912, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1750351872, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1750376448, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1832951808, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1902157824, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1971363840, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1971388416, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 1975713792, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 2001666048, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 2027618304, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 2032779264, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2032803840, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 2115379200, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 2184585216, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 2253791232, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 2253815808, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 2258141184, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 2284093440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 2310045696, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 2314371072, "shape": [ 6144, 1024 ], "size": 4325376, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 2318696448, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 2344648704, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2348974080, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2348998656, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 2418204672, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 2418229248, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2418253824, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 2487459840, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 2556665856, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 2625871872, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2625896448, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 2630221824, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 2656174080, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 2682126336, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 2687287296, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2687311872, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2769887232, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2839093248, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 2908299264, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2908323840, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 2912649216, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 2938601472, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 2964553728, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2968879104, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2968903680, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 3038109696, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 3107315712, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 3176521728, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3176546304, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 3180871680, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 3206823936, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 3232776192, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 3237101568, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3237126144, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 3306332160, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 3375538176, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 3444744192, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3444768768, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 3449094144, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 3475046400, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.9.attn_v.weight", "offset": 3500998656, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 3506159616, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3506184192, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 3588759552, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 3657965568, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 3727171584, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3727196160, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 3753148416, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3753172992, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 3822379008, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 3891585024, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 3960791040, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3960815616, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 3965140992, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 3991093248, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 4017045504, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 4021370880, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4021395456, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 4090601472, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 4159807488, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 4229013504, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 4229038080, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 4233363456, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 4259315712, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 4285267968, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 4289593344, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4289617920, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 4372193280, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 4441399296, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 4510605312, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 4510629888, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 4514955264, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 4540907520, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 4566859776, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4572020736, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4572045312, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 4641251328, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 4710457344, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 4779663360, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4779687936, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 4784013312, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 4809965568, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 4835917824, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 4840243200, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4840267776, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 4909473792, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 4978679808, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 5047885824, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 5047910400, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 5052235776, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 5078188032, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.25.attn_k.w", "offset": 5104140288, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 5108465664, "shape": [ 6144, 16384 ], "size": 4325376, "type": 13 }, { "name": "blk.18.attn_k.weight", "offset": 5177671680, "shape": [ 6144, 1024 ], "size": 69206016, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 5181997056, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 5207949312, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 5233901568, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 5239062528, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5239087104, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 5321662464, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 5390868480, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 5390893056, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5390917632, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 5460123648, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 5529329664, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 5598535680, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 5598560256, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 5602885632, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 5628837888, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 5654790144, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 5659115520, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5659140096, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 5728346112, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 5797552128, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 5866758144, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 5866782720, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 5871108096, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 5897060352, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 5923012608, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 5927337984, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5927362560, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 6009937920, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 6079143936, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 6148349952, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 6148374528, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 6152699904, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 6178652160, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 6204604416, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 6209765376, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6209789952, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 6278995968, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 6348201984, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 6417408000, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 6417432576, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 6421757952, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 6447710208, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 6473662464, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 6477987840, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6478012416, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 6547218432, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 6616424448, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 6685630464, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 6685655040, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 6689980416, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 6715932672, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 6741884928, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 6746210304, "shape": [ 6144, 16384 ], "size": 4325376, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 6815416320, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.24.attn_k.weight", "offset": 6884622336, "shape": [ 6144, 1024 ], "size": 69206016, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 6888947712, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 6914899968, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 6940852224, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 6946013184, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6946037760, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 7028613120, "shape": [ 6144 ], "size": 82575360, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 7028637696, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7028662272, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 7097868288, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 7167074304, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 7236280320, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 7236304896, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 7240630272, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 7266582528, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 7292534784, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 7296860160, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7296884736, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 7366090752, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 7435296768, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 7504502784, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 7504527360, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 7508852736, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 7534804992, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 7560757248, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 7565082624, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7565107200, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 7647682560, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 7716888576, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 7786094592, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 7786119168, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 7790444544, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 7816396800, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 7842349056, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 7847510016, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7847534592, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 7916740608, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 7985946624, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 8055152640, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 8055177216, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 8059502592, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 8085454848, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 8111407104, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 8115732480, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 8115757056, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 8184963072, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 8254169088, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 8323375104, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 8323399680, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 8327725056, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 8353677312, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 8379629568, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 8383954944, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 8383979520, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.30.ffn_gate.weight", "offset": 8466554880, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 8535760896, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 8604966912, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 8604991488, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 8609316864, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 8635269120, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 8661221376, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.31.attn_k.weight", "offset": 8666382336, "shape": [ 6144, 1024 ], "size": 5160960, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 8670707712, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 8696659968, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 8700985344, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8701009920, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 8770215936, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 8839421952, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 8908627968, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 8908652544, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 8934604800, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 8934629376, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 9003835392, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 9073041408, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 9142247424, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 9142272000, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 9146597376, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 9172549632, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 9198501888, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 9202827264, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 9202851840, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 9285427200, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 9354633216, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 9423839232, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 9423863808, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 9428189184, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 9454141440, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 9480093696, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 9485254656, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 9485279232, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 9554485248, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 9623691264, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 9692897280, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 9692921856, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 9697247232, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 9723199488, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 9749151744, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 9753477120, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 9753501696, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 9822707712, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 9891913728, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 9961119744, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 9961144320, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 9965469696, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 9991421952, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 10017374208, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 10021699584, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 10021724160, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 10104299520, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 10173505536, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 10242711552, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 10242736128, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 10247061504, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 10273013760, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 10298966016, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 10304126976, "shape": [ 6144, 16384 ], "size": 5160960, "type": 13 }, { "name": "blk.37.attn_k.weight", "offset": 10373332992, "shape": [ 6144, 1024 ], "size": 69206016, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 10377658368, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 10403610624, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 10429562880, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 10433888256, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 10433912832, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 10503118848, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 10572324864, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 10572349440, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 10572374016, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 10641580032, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 10710786048, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 10779992064, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 10780016640, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 10784342016, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 10810294272, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 10836246528, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 10840571904, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 10840596480, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 10923171840, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 10992377856, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 11061583872, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 11061608448, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 11065933824, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 11091886080, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 11117838336, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 11122999296, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 11123023872, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 11192229888, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 11261435904, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 11330641920, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 11330666496, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 11334991872, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 11360944128, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 11386896384, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 11391221760, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 11391246336, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 11460452352, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 11529658368, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 11598864384, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 11598888960, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 11603214336, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 11629166592, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 11655118848, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.42.attn_norm.weight", "offset": 11659444224, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 11659468800, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.42.ffn_gate.weight", "offset": 11742044160, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.42.ffn_up.weight", "offset": 11811250176, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 11880456192, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 11880480768, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 11884806144, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 11910758400, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.42.attn_v.weight", "offset": 11936710656, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.43.ffn_gate.weight", "offset": 11941871616, "shape": [ 6144, 16384 ], "size": 5160960, "type": 13 }, { "name": "blk.43.ffn_up.weight", "offset": 12011077632, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.43.attn_k.weight", "offset": 12080283648, "shape": [ 6144, 1024 ], "size": 69206016, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 12084609024, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 12110561280, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.43.attn_v.weight", "offset": 12136513536, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 12140838912, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 12140863488, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 12210069504, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 12210094080, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 12210118656, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.44.ffn_gate.weight", "offset": 12279324672, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.44.ffn_up.weight", "offset": 12348530688, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 12417736704, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 12417761280, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 12422086656, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 12448038912, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.44.attn_v.weight", "offset": 12473991168, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 12478316544, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 12478341120, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.45.ffn_gate.weight", "offset": 12560916480, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.45.ffn_up.weight", "offset": 12630122496, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.45.ffn_norm.weight", "offset": 12699328512, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 12699353088, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 12703678464, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 12729630720, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.45.attn_v.weight", "offset": 12755582976, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.46.attn_norm.weight", "offset": 12760743936, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 12760768512, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.46.ffn_gate.weight", "offset": 12829974528, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.46.ffn_up.weight", "offset": 12899180544, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 12968386560, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 12968411136, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 12972736512, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 12998688768, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.46.attn_v.weight", "offset": 13024641024, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 13028966400, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 13028990976, "shape": [ 16384, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.47.ffn_gate.weight", "offset": 13098196992, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.47.ffn_up.weight", "offset": 13167403008, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 13236609024, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 13236633600, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 13240958976, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 13266911232, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.47.attn_v.weight", "offset": 13292863488, "shape": [ 6144, 1024 ], "size": 25952256, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 13297188864, "shape": [ 6144 ], "size": 4325376, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 13297213440, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.48.ffn_gate.weight", "offset": 13379788800, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 13448994816, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.48.ffn_norm.weight", "offset": 13518200832, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 13518225408, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.48.attn_output.weight", "offset": 13522550784, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.48.attn_q.weight", "offset": 13548503040, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.48.attn_v.weight", "offset": 13574455296, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 13579616256, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 13579640832, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.49.ffn_gate.weight", "offset": 13662216192, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.49.ffn_up.weight", "offset": 13731422208, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.49.ffn_norm.weight", "offset": 13800628224, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 13800652800, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.49.attn_output.weight", "offset": 13804978176, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.49.attn_q.weight", "offset": 13830930432, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.49.attn_v.weight", "offset": 13856882688, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.50.attn_k.weight", "offset": 13862043648, "shape": [ 6144, 1024 ], "size": 5160960, "type": 13 }, { "name": "blk.50.attn_q.weight", "offset": 13866369024, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.50.attn_v.weight", "offset": 13892321280, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "output.weight", "offset": 13897482240, "shape": [ 6144, 32768 ], "size": 5160960, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 14062632960, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 14062657536, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.50.ffn_gate.weight", "offset": 14145232896, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.50.ffn_up.weight", "offset": 14214438912, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.50.ffn_norm.weight", "offset": 14283644928, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 14283669504, "shape": [ 6144, 6144 ], "size": 24576, "type": 13 }, { "name": "blk.51.attn_norm.weight", "offset": 14309621760, "shape": [ 6144 ], "size": 25952256, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 14309646336, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.51.ffn_gate.weight", "offset": 14392221696, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.51.ffn_up.weight", "offset": 14461427712, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.51.ffn_norm.weight", "offset": 14530633728, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 14530658304, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.51.attn_output.weight", "offset": 14534983680, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.51.attn_q.weight", "offset": 14560935936, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.51.attn_v.weight", "offset": 14586888192, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.52.attn_norm.weight", "offset": 14592049152, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 14592073728, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.52.ffn_gate.weight", "offset": 14674649088, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.52.ffn_up.weight", "offset": 14743855104, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.52.ffn_norm.weight", "offset": 14813061120, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 14813085696, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.52.attn_output.weight", "offset": 14817411072, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.52.attn_q.weight", "offset": 14843363328, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.52.attn_v.weight", "offset": 14869315584, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.53.attn_norm.weight", "offset": 14874476544, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 14874501120, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.53.ffn_gate.weight", "offset": 14957076480, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.53.ffn_up.weight", "offset": 15026282496, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.53.ffn_norm.weight", "offset": 15095488512, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 15095513088, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.53.attn_output.weight", "offset": 15099838464, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.53.attn_q.weight", "offset": 15125790720, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.53.attn_v.weight", "offset": 15151742976, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 15156903936, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 15156928512, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.54.ffn_gate.weight", "offset": 15239503872, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.54.ffn_up.weight", "offset": 15308709888, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.54.ffn_norm.weight", "offset": 15377915904, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 15377940480, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.54.attn_output.weight", "offset": 15382265856, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.54.attn_q.weight", "offset": 15408218112, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.54.attn_v.weight", "offset": 15434170368, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "blk.55.attn_norm.weight", "offset": 15439331328, "shape": [ 6144 ], "size": 5160960, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 15439355904, "shape": [ 16384, 6144 ], "size": 24576, "type": 14 }, { "name": "blk.55.ffn_gate.weight", "offset": 15521931264, "shape": [ 6144, 16384 ], "size": 82575360, "type": 13 }, { "name": "blk.55.ffn_up.weight", "offset": 15591137280, "shape": [ 6144, 16384 ], "size": 69206016, "type": 13 }, { "name": "blk.55.ffn_norm.weight", "offset": 15660343296, "shape": [ 6144 ], "size": 69206016, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 15660367872, "shape": [ 6144, 1024 ], "size": 24576, "type": 13 }, { "name": "blk.55.attn_output.weight", "offset": 15664693248, "shape": [ 6144, 6144 ], "size": 4325376, "type": 13 }, { "name": "blk.55.attn_q.weight", "offset": 15690645504, "shape": [ 6144, 6144 ], "size": 25952256, "type": 13 }, { "name": "blk.55.attn_v.weight", "offset": 15716597760, "shape": [ 6144, 1024 ], "size": 25952256, "type": 14 }, { "name": "output_norm.weight", "offset": 15721758720, "shape": [ 6144 ], "size": 5160960, "type": 0 } ], "version": 3 }