mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 12, "general.name": "mistralai", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 32768, "llama.embedding_length": 4096, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.merges": "... (58980 values)", "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 46702792704, "tensors": [ { "name": "blk.0.ffn_gate.0.weight", "offset": 56320000, "shape": [ 4096, 14336 ], "size": 56320000, "type": 11 }, { "name": "blk.0.ffn_down.0.weight", "offset": 81551360, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.0.weight", "offset": 106782720, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.1.weight", "offset": 132014080, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.1.weight", "offset": 157245440, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.1.weight", "offset": 182476800, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.2.weight", "offset": 207708160, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.2.weight", "offset": 232939520, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.2.weight", "offset": 258170880, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.3.weight", "offset": 283402240, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.3.weight", "offset": 308633600, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.3.weight", "offset": 333864960, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.4.weight", "offset": 359096320, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.4.weight", "offset": 384327680, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.4.weight", "offset": 409559040, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.5.weight", "offset": 434790400, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.5.weight", "offset": 460021760, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.5.weight", "offset": 485253120, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.6.weight", "offset": 510484480, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.6.weight", "offset": 535715840, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.6.weight", "offset": 560947200, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate.7.weight", "offset": 586178560, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_down.7.weight", "offset": 611409920, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_up.7.weight", "offset": 636641280, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 661872640, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 661938176, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 661954560, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 661970944, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 666427392, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 675864576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 683073536, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.1.ffn_gate.0.weight", "offset": 687529984, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.1.ffn_down.0.weight", "offset": 712761344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.0.weight", "offset": 737992704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.1.weight", "offset": 763224064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.1.weight", "offset": 788455424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.1.weight", "offset": 813686784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.2.weight", "offset": 838918144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.2.weight", "offset": 864149504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.2.weight", "offset": 889380864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.3.weight", "offset": 914612224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.3.weight", "offset": 939843584, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.3.weight", "offset": 965074944, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.4.weight", "offset": 990306304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.4.weight", "offset": 1015537664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1040769024, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1040834560, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1045291008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 1054728192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 1061937152, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.1.ffn_up.4.weight", "offset": 1066393600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.1.ffn_gate.5.weight", "offset": 1091624960, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.5.weight", "offset": 1116856320, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.5.weight", "offset": 1142087680, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.6.weight", "offset": 1167319040, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.6.weight", "offset": 1192550400, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.6.weight", "offset": 1217781760, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_gate.7.weight", "offset": 1243013120, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_down.7.weight", "offset": 1268244480, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.1.ffn_up.7.weight", "offset": 1293475840, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 1318707200, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 1318723584, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.ffn_gate.0.weight", "offset": 1318739968, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.2.ffn_down.0.weight", "offset": 1343971328, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.0.weight", "offset": 1369202688, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.1.weight", "offset": 1394434048, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.1.weight", "offset": 1419665408, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.1.weight", "offset": 1444896768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.2.weight", "offset": 1470128128, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.2.weight", "offset": 1495359488, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.2.weight", "offset": 1520590848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.3.weight", "offset": 1545822208, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.3.weight", "offset": 1571053568, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.3.weight", "offset": 1596284928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.4.weight", "offset": 1621516288, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.4.weight", "offset": 1646747648, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.4.weight", "offset": 1671979008, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.5.weight", "offset": 1697210368, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.5.weight", "offset": 1722441728, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.5.weight", "offset": 1747673088, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.6.weight", "offset": 1772904448, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.6.weight", "offset": 1798135808, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.6.weight", "offset": 1823367168, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate.7.weight", "offset": 1848598528, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_down.7.weight", "offset": 1873829888, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_up.7.weight", "offset": 1899061248, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1924292608, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1924358144, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 1924374528, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1924390912, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1928847360, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 1938284544, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 1945493504, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.3.ffn_gate.0.weight", "offset": 1949949952, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.3.ffn_down.0.weight", "offset": 1975181312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.0.weight", "offset": 2000412672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.1.weight", "offset": 2025644032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.1.weight", "offset": 2050875392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.1.weight", "offset": 2076106752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.2.weight", "offset": 2101338112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 2126569472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 2126635008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 2131091456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 2140528640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 2147737600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.3.ffn_down.2.weight", "offset": 2152194048, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.3.ffn_up.2.weight", "offset": 2177425408, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.3.weight", "offset": 2202656768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.3.weight", "offset": 2227888128, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.3.weight", "offset": 2253119488, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.4.weight", "offset": 2278350848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.4.weight", "offset": 2303582208, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.4.weight", "offset": 2328813568, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.5.weight", "offset": 2354044928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.5.weight", "offset": 2379276288, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.5.weight", "offset": 2404507648, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.6.weight", "offset": 2429739008, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.6.weight", "offset": 2454970368, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.6.weight", "offset": 2480201728, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_gate.7.weight", "offset": 2505433088, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_down.7.weight", "offset": 2530664448, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.3.ffn_up.7.weight", "offset": 2555895808, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 2581127168, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 2581143552, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.ffn_gate.0.weight", "offset": 2581159936, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.4.ffn_down.0.weight", "offset": 2606391296, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.0.weight", "offset": 2631622656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.1.weight", "offset": 2656854016, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.1.weight", "offset": 2682085376, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.1.weight", "offset": 2707316736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.2.weight", "offset": 2732548096, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.2.weight", "offset": 2757779456, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.2.weight", "offset": 2783010816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.3.weight", "offset": 2808242176, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.3.weight", "offset": 2833473536, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.3.weight", "offset": 2858704896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.4.weight", "offset": 2883936256, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.4.weight", "offset": 2909167616, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.4.weight", "offset": 2934398976, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.5.weight", "offset": 2959630336, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.5.weight", "offset": 2984861696, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.5.weight", "offset": 3010093056, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.8.ffn_up.4.we", "offset": 3035324416, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.6.weight", "offset": 3060555776, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.6.weight", "offset": 3085787136, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate.7.weight", "offset": 3111018496, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_down.7.weight", "offset": 3136249856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_up.7.weight", "offset": 3161481216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 3186712576, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3186778112, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 3186794496, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 3186810880, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 3191267328, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 3200704512, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 3207913472, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 3212369920, "shape": [ 4096, 8 ], "size": 4456448, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 3212435456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 3216891904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 3226329088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 3233538048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.5.ffn_gate.0.weight", "offset": 3237994496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.5.ffn_down.0.weight", "offset": 3263225856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.0.weight", "offset": 3288457216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.1.weight", "offset": 3313688576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.1.weight", "offset": 3338919936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.1.weight", "offset": 3364151296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.2.weight", "offset": 3389382656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.2.weight", "offset": 3414614016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.2.weight", "offset": 3439845376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.3.weight", "offset": 3465076736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.3.weight", "offset": 3490308096, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.3.weight", "offset": 3515539456, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.4.weight", "offset": 3540770816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.4.weight", "offset": 3566002176, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.4.weight", "offset": 3591233536, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.5.weight", "offset": 3616464896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.5.weight", "offset": 3641696256, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.5.weight", "offset": 3666927616, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.6.weight", "offset": 3692158976, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.6.weight", "offset": 3717390336, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.6.weight", "offset": 3742621696, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_gate.7.weight", "offset": 3767853056, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_down.7.weight", "offset": 3793084416, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.5.ffn_up.7.weight", "offset": 3818315776, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 3843547136, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 3843563520, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.ffn_gate.0.weight", "offset": 3843579904, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.6.ffn_down.0.weight", "offset": 3868811264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.0.weight", "offset": 3894042624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.1.weight", "offset": 3919273984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.1.weight", "offset": 3944505344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.1.weight", "offset": 3969736704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.2.weight", "offset": 3994968064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.2.weight", "offset": 4020199424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.2.weight", "offset": 4045430784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.3.weight", "offset": 4070662144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.3.weight", "offset": 4095893504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.3.weight", "offset": 4121124864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.4.weight", "offset": 4146356224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.4.weight", "offset": 4171587584, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.4.weight", "offset": 4196818944, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.5.weight", "offset": 4222050304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.5.weight", "offset": 4247281664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 4272513024, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 4272578560, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 4277035008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 4286472192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 4293681152, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.6.ffn_up.5.weight", "offset": 4298137600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.6.ffn_gate.6.weight", "offset": 4323368960, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.6.weight", "offset": 4348600320, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.6.weight", "offset": 4373831680, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_gate.7.weight", "offset": 4399063040, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_down.7.weight", "offset": 4424294400, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.6.ffn_up.7.weight", "offset": 4449525760, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 4474757120, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 4474773504, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.ffn_gate.0.weight", "offset": 4474789888, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.7.ffn_down.0.weight", "offset": 4500021248, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.0.weight", "offset": 4525252608, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.1.weight", "offset": 4550483968, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.1.weight", "offset": 4575715328, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.1.weight", "offset": 4600946688, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.2.weight", "offset": 4626178048, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.2.weight", "offset": 4651409408, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.2.weight", "offset": 4676640768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.3.weight", "offset": 4701872128, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.3.weight", "offset": 4727103488, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.3.weight", "offset": 4752334848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.4.weight", "offset": 4777566208, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.4.weight", "offset": 4802797568, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.4.weight", "offset": 4828028928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.5.weight", "offset": 4853260288, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.5.weight", "offset": 4878491648, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.5.weight", "offset": 4903723008, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.6.weight", "offset": 4928954368, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.6.weight", "offset": 4954185728, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.6.weight", "offset": 4979417088, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate.7.weight", "offset": 5004648448, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_down.7.weight", "offset": 5029879808, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_up.7.weight", "offset": 5055111168, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 5080342528, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 5080408064, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 5080424448, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 5080440832, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 5084897280, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 5094334464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 5101543424, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.8.ffn_gate.0.weight", "offset": 5105999872, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.8.ffn_down.0.weight", "offset": 5131231232, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.0.weight", "offset": 5156462592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.1.weight", "offset": 5181693952, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.1.weight", "offset": 5206925312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.1.weight", "offset": 5232156672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.2.weight", "offset": 5257388032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.2.weight", "offset": 5282619392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.2.weight", "offset": 5307850752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.3.weight", "offset": 5333082112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 5358313472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 5358379008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 5362835456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 5372272640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 5379481600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.10.ffn_gate.0.weight", "offset": 5383938048, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.10.ffn_down.0.weight", "offset": 5409169408, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.0.weight", "offset": 5434400768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 5459632128, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 5459697664, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 5464154112, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 5473591296, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 5480800256, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.8.ffn_down.3.weight", "offset": 5485256704, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.8.ffn_up.3.weight", "offset": 5510488064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.4.weight", "offset": 5535719424, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.4.weight", "offset": 5560950784, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.4.weight", "offset": 5586182144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.5.weight", "offset": 5611413504, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.5.weight", "offset": 5636644864, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.5.weight", "offset": 5661876224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.6.weight", "offset": 5687107584, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.6.weight", "offset": 5712338944, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.6.weight", "offset": 5737570304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_gate.7.weight", "offset": 5762801664, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_down.7.weight", "offset": 5788033024, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.8.ffn_up.7.weight", "offset": 5813264384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 5838495744, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 5838512128, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.ffn_gate.0.weight", "offset": 5838528512, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.9.ffn_down.0.weight", "offset": 5863759872, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.0.weight", "offset": 5888991232, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.1.weight", "offset": 5914222592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.1.weight", "offset": 5939453952, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.1.weight", "offset": 5964685312, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.2.weight", "offset": 5989916672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.2.weight", "offset": 6015148032, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.2.weight", "offset": 6040379392, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.3.weight", "offset": 6065610752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.3.weight", "offset": 6090842112, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.3.weight", "offset": 6116073472, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.4.weight", "offset": 6141304832, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.4.weight", "offset": 6166536192, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.4.weight", "offset": 6191767552, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.5.weight", "offset": 6216998912, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.5.weight", "offset": 6242230272, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.5.weight", "offset": 6267461632, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.6.weight", "offset": 6292692992, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.6.weight", "offset": 6317924352, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.6.weight", "offset": 6343155712, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate.7.weight", "offset": 6368387072, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_down.7.weight", "offset": 6393618432, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_up.7.weight", "offset": 6418849792, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 6444081152, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 6444146688, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 6444163072, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 6444179456, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 6448635904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 6458073088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 6465282048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.10.ffn_gate.1.weight", "offset": 6469738496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.10.ffn_down.1.weight", "offset": 6494969856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.1.weight", "offset": 6520201216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.2.weight", "offset": 6545432576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.2.weight", "offset": 6570663936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.2.weight", "offset": 6595895296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.3.weight", "offset": 6621126656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.3.weight", "offset": 6646358016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.3.weight", "offset": 6671589376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.4.weight", "offset": 6696820736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.4.weight", "offset": 6722052096, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.4.weight", "offset": 6747283456, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.5.weight", "offset": 6772514816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.5.weight", "offset": 6797746176, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.5.weight", "offset": 6822977536, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.6.weight", "offset": 6848208896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.6.weight", "offset": 6873440256, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.6.weight", "offset": 6898671616, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_gate.7.weight", "offset": 6923902976, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_down.7.weight", "offset": 6949134336, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.10.ffn_up.7.weight", "offset": 6974365696, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 6999597056, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 6999613440, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.ffn_gate.0.weight", "offset": 6999629824, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.11.ffn_down.0.weight", "offset": 7024861184, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.0.weight", "offset": 7050092544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.1.weight", "offset": 7075323904, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.1.weight", "offset": 7100555264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.1.weight", "offset": 7125786624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.2.weight", "offset": 7151017984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.2.weight", "offset": 7176249344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.2.weight", "offset": 7201480704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.3.weight", "offset": 7226712064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.3.weight", "offset": 7251943424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.3.weight", "offset": 7277174784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.4.weight", "offset": 7302406144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.4.weight", "offset": 7327637504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.4.weight", "offset": 7352868864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.5.weight", "offset": 7378100224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.5.weight", "offset": 7403331584, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.5.weight", "offset": 7428562944, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate.6.weight", "offset": 7453794304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.6.weight", "offset": 7479025664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 7504257024, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 7504322560, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 7508779008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 7518216192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 7525425152, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.11.ffn_up.6.weight", "offset": 7529881600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.11.ffn_gate.7.weight", "offset": 7555112960, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_down.7.weight", "offset": 7580344320, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.11.ffn_up.7.weight", "offset": 7605575680, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 7630807040, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 7630823424, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.ffn_gate.0.weight", "offset": 7630839808, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.12.ffn_down.0.weight", "offset": 7656071168, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.0.weight", "offset": 7681302528, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.1.weight", "offset": 7706533888, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.1.weight", "offset": 7731765248, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.1.weight", "offset": 7756996608, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.2.weight", "offset": 7782227968, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.2.weight", "offset": 7807459328, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.2.weight", "offset": 7832690688, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.3.weight", "offset": 7857922048, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.3.weight", "offset": 7883153408, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.3.weight", "offset": 7908384768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.4.weight", "offset": 7933616128, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.4.weight", "offset": 7958847488, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.4.weight", "offset": 7984078848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.5.weight", "offset": 8009310208, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.5.weight", "offset": 8034541568, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.5.weight", "offset": 8059772928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.6.weight", "offset": 8085004288, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.6.weight", "offset": 8110235648, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.6.weight", "offset": 8135467008, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate.7.weight", "offset": 8160698368, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_down.7.weight", "offset": 8185929728, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_up.7.weight", "offset": 8211161088, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 8236392448, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 8236457984, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 8236474368, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 8236490752, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 8240947200, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 8250384384, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 8257593344, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.13.ffn_gate.0.weight", "offset": 8262049792, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.13.ffn_down.0.weight", "offset": 8287281152, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.0.weight", "offset": 8312512512, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.1.weight", "offset": 8337743872, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.1.weight", "offset": 8362975232, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.1.weight", "offset": 8388206592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.2.weight", "offset": 8413437952, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.2.weight", "offset": 8438669312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.2.weight", "offset": 8463900672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.3.weight", "offset": 8489132032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.3.weight", "offset": 8514363392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.3.weight", "offset": 8539594752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.4.weight", "offset": 8564826112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 8590057472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 8590123008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 8594579456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 8604016640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 8611225600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.13.ffn_down.4.weight", "offset": 8615682048, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.13.ffn_up.4.weight", "offset": 8640913408, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.5.weight", "offset": 8666144768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.5.weight", "offset": 8691376128, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.5.weight", "offset": 8716607488, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.6.weight", "offset": 8741838848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.6.weight", "offset": 8767070208, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.6.weight", "offset": 8792301568, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_gate.7.weight", "offset": 8817532928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_down.7.weight", "offset": 8842764288, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.13.ffn_up.7.weight", "offset": 8867995648, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 8893227008, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 8893243392, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.ffn_gate.0.weight", "offset": 8893259776, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.14.ffn_down.0.weight", "offset": 8918491136, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.0.weight", "offset": 8943722496, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.1.weight", "offset": 8968953856, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.1.weight", "offset": 8994185216, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.1.weight", "offset": 9019416576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.2.weight", "offset": 9044647936, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.2.weight", "offset": 9069879296, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.2.weight", "offset": 9095110656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.3.weight", "offset": 9120342016, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.3.weight", "offset": 9145573376, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.3.weight", "offset": 9170804736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.4.weight", "offset": 9196036096, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.4.weight", "offset": 9221267456, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.4.weight", "offset": 9246498816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.5.weight", "offset": 9271730176, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.5.weight", "offset": 9296961536, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.5.weight", "offset": 9322192896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.6.weight", "offset": 9347424256, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.6.weight", "offset": 9372655616, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.6.weight", "offset": 9397886976, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate.7.weight", "offset": 9423118336, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_down.7.weight", "offset": 9448349696, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_up.7.weight", "offset": 9473581056, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 9498812416, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 9498877952, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 9498894336, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 9498910720, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 9503367168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 9512804352, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 9520013312, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.15.ffn_gate.0.weight", "offset": 9524469760, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.15.ffn_down.0.weight", "offset": 9549701120, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.0.weight", "offset": 9574932480, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.1.weight", "offset": 9600163840, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.1.weight", "offset": 9625395200, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.1.weight", "offset": 9650626560, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 9675857920, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 9675923456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 9680379904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 9689817088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 9697026048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.15.ffn_gate.2.weight", "offset": 9701482496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.15.ffn_down.2.weight", "offset": 9726713856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.2.weight", "offset": 9751945216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.3.weight", "offset": 9777176576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.3.weight", "offset": 9802407936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.3.weight", "offset": 9827639296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.4.weight", "offset": 9852870656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.4.weight", "offset": 9878102016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.4.weight", "offset": 9903333376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.5.weight", "offset": 9928564736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.5.weight", "offset": 9953796096, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.5.weight", "offset": 9979027456, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.6.weight", "offset": 10004258816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.6.weight", "offset": 10029490176, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.6.weight", "offset": 10054721536, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_gate.7.weight", "offset": 10079952896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_down.7.weight", "offset": 10105184256, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.15.ffn_up.7.weight", "offset": 10130415616, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 10155646976, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 10155663360, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.ffn_gate.0.weight", "offset": 10155679744, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.16.ffn_down.0.weight", "offset": 10180911104, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.0.weight", "offset": 10206142464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.1.weight", "offset": 10231373824, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.1.weight", "offset": 10256605184, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.1.weight", "offset": 10281836544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.2.weight", "offset": 10307067904, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.2.weight", "offset": 10332299264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.2.weight", "offset": 10357530624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.3.weight", "offset": 10382761984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.3.weight", "offset": 10407993344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.3.weight", "offset": 10433224704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.4.weight", "offset": 10458456064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.4.weight", "offset": 10483687424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.4.weight", "offset": 10508918784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.5.weight", "offset": 10534150144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.5.weight", "offset": 10559381504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.5.weight", "offset": 10584612864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.6.weight", "offset": 10609844224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.6.weight", "offset": 10635075584, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_up.6.weight", "offset": 10660306944, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate.7.weight", "offset": 10685538304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_down.7.weight", "offset": 10710769664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 10736001024, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 10736066560, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 10740523008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 10749960192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 10757169152, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.16.ffn_up.7.weight", "offset": 10761625600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 10786856960, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 10786873344, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.ffn_gate.0.weight", "offset": 10786889728, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.17.ffn_down.0.weight", "offset": 10812121088, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.0.weight", "offset": 10837352448, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.1.weight", "offset": 10862583808, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.1.weight", "offset": 10887815168, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.1.weight", "offset": 10913046528, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.2.weight", "offset": 10938277888, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.2.weight", "offset": 10963509248, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.2.weight", "offset": 10988740608, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.3.weight", "offset": 11013971968, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.3.weight", "offset": 11039203328, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.3.weight", "offset": 11064434688, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.4.weight", "offset": 11089666048, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.4.weight", "offset": 11114897408, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.4.weight", "offset": 11140128768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.5.weight", "offset": 11165360128, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.5.weight", "offset": 11190591488, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.5.weight", "offset": 11215822848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.6.weight", "offset": 11241054208, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.6.weight", "offset": 11266285568, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.6.weight", "offset": 11291516928, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate.7.weight", "offset": 11316748288, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_down.7.weight", "offset": 11341979648, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_up.7.weight", "offset": 11367211008, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 11392442368, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 11392507904, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 11392524288, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 11392540672, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 11396997120, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 11406434304, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 11413643264, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.18.ffn_gate.0.weight", "offset": 11418099712, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.18.ffn_down.0.weight", "offset": 11443331072, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.0.weight", "offset": 11468562432, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.1.weight", "offset": 11493793792, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.1.weight", "offset": 11519025152, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.1.weight", "offset": 11544256512, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.2.weight", "offset": 11569487872, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.2.weight", "offset": 11594719232, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.2.weight", "offset": 11619950592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.3.weight", "offset": 11645181952, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.3.weight", "offset": 11670413312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.3.weight", "offset": 11695644672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.4.weight", "offset": 11720876032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.4.weight", "offset": 11746107392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.4.weight", "offset": 11771338752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.5.weight", "offset": 11796570112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 11821801472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 11821867008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 11826323456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 11835760640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 11842969600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.18.ffn_down.5.weight", "offset": 11847426048, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.18.ffn_up.5.weight", "offset": 11872657408, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.6.weight", "offset": 11897888768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.6.weight", "offset": 11923120128, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.6.weight", "offset": 11948351488, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_gate.7.weight", "offset": 11973582848, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_down.7.weight", "offset": 11998814208, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.18.ffn_up.7.weight", "offset": 12024045568, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 12049276928, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 12049293312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.ffn_gate.0.weight", "offset": 12049309696, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.19.ffn_down.0.weight", "offset": 12074541056, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.0.weight", "offset": 12099772416, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.1.weight", "offset": 12125003776, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.1.weight", "offset": 12150235136, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.1.weight", "offset": 12175466496, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.2.weight", "offset": 12200697856, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.2.weight", "offset": 12225929216, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.2.weight", "offset": 12251160576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.3.weight", "offset": 12276391936, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.3.weight", "offset": 12301623296, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.3.weight", "offset": 12326854656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.4.weight", "offset": 12352086016, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.4.weight", "offset": 12377317376, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.4.weight", "offset": 12402548736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.5.weight", "offset": 12427780096, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.5.weight", "offset": 12453011456, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.5.weight", "offset": 12478242816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.6.weight", "offset": 12503474176, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.6.weight", "offset": 12528705536, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.6.weight", "offset": 12553936896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate.7.weight", "offset": 12579168256, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_down.7.weight", "offset": 12604399616, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_up.7.weight", "offset": 12629630976, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 12654862336, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 12654927872, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 12654944256, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 12654960640, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 12659417088, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 12668854272, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 12676063232, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.20.ffn_gate.0.weight", "offset": 12680519680, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.20.ffn_down.0.weight", "offset": 12705751040, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.0.weight", "offset": 12730982400, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.1.weight", "offset": 12756213760, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.1.weight", "offset": 12781445120, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.1.weight", "offset": 12806676480, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.2.weight", "offset": 12831907840, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.2.weight", "offset": 12857139200, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.2.weight", "offset": 12882370560, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 12907601920, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 12907667456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 12912123904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 12921561088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 12928770048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.20.ffn_gate.3.weight", "offset": 12933226496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.20.ffn_down.3.weight", "offset": 12958457856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.3.weight", "offset": 12983689216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.4.weight", "offset": 13008920576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.4.weight", "offset": 13034151936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.4.weight", "offset": 13059383296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.5.weight", "offset": 13084614656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.5.weight", "offset": 13109846016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.5.weight", "offset": 13135077376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.6.weight", "offset": 13160308736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.6.weight", "offset": 13185540096, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.6.weight", "offset": 13210771456, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_gate.7.weight", "offset": 13236002816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_down.7.weight", "offset": 13261234176, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.20.ffn_up.7.weight", "offset": 13286465536, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 13311696896, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 13311713280, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_gate.0.weight", "offset": 13311729664, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.21.ffn_down.0.weight", "offset": 13336961024, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.0.weight", "offset": 13362192384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.1.weight", "offset": 13387423744, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.1.weight", "offset": 13412655104, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.1.weight", "offset": 13437886464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.2.weight", "offset": 13463117824, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.2.weight", "offset": 13488349184, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.2.weight", "offset": 13513580544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.3.weight", "offset": 13538811904, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.3.weight", "offset": 13564043264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.3.weight", "offset": 13589274624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.4.weight", "offset": 13614505984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.4.weight", "offset": 13639737344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.4.weight", "offset": 13664968704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.5.weight", "offset": 13690200064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.5.weight", "offset": 13715431424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.25.ffn_down", "offset": 13740662784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.6.weight", "offset": 13765894144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.6.weight", "offset": 13791125504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.6.weight", "offset": 13816356864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate.7.weight", "offset": 13841588224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_down.7.weight", "offset": 13866819584, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_up.7.weight", "offset": 13892050944, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 13917282304, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 13917347840, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 13917364224, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 13917380608, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 13921837056, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 13931274240, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 13938483200, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.22.ffn_gate.0.weight", "offset": 13942939648, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.22.ffn_down.0.weight", "offset": 13968171008, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 13993402368, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 13993467904, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 13997924352, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 14007361536, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 14014570496, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.22.ffn_up.0.weight", "offset": 14019026944, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.22.ffn_gate.1.weight", "offset": 14044258304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.1.weight", "offset": 14069489664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.1.weight", "offset": 14094721024, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.2.weight", "offset": 14119952384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.2.weight", "offset": 14145183744, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.2.weight", "offset": 14170415104, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.3.weight", "offset": 14195646464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.3.weight", "offset": 14220877824, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.3.weight", "offset": 14246109184, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.4.weight", "offset": 14271340544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.4.weight", "offset": 14296571904, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.4.weight", "offset": 14321803264, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.5.weight", "offset": 14347034624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.5.weight", "offset": 14372265984, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.5.weight", "offset": 14397497344, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.6.weight", "offset": 14422728704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.6.weight", "offset": 14447960064, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.6.weight", "offset": 14473191424, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_gate.7.weight", "offset": 14498422784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_down.7.weight", "offset": 14523654144, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.22.ffn_up.7.weight", "offset": 14548885504, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 14574116864, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 14574133248, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.ffn_gate.0.weight", "offset": 14574149632, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.23.ffn_down.0.weight", "offset": 14599380992, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.0.weight", "offset": 14624612352, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.1.weight", "offset": 14649843712, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.1.weight", "offset": 14675075072, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.1.weight", "offset": 14700306432, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.2.weight", "offset": 14725537792, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.2.weight", "offset": 14750769152, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.2.weight", "offset": 14776000512, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.3.weight", "offset": 14801231872, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.3.weight", "offset": 14826463232, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.3.weight", "offset": 14851694592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.4.weight", "offset": 14876925952, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.4.weight", "offset": 14902157312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.4.weight", "offset": 14927388672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.5.weight", "offset": 14952620032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.5.weight", "offset": 14977851392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.5.weight", "offset": 15003082752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.6.weight", "offset": 15028314112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 15053545472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.23.attn_k.weight", "offset": 15053611008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 15058067456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 15067504640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 15074713600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.23.ffn_down.6.weight", "offset": 15079170048, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.23.ffn_up.6.weight", "offset": 15104401408, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_gate.7.weight", "offset": 15129632768, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_down.7.weight", "offset": 15154864128, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.23.ffn_up.7.weight", "offset": 15180095488, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 15205326848, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 15205343232, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.ffn_gate.0.weight", "offset": 15205359616, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.24.ffn_down.0.weight", "offset": 15230590976, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.0.weight", "offset": 15255822336, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.1.weight", "offset": 15281053696, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.1.weight", "offset": 15306285056, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.1.weight", "offset": 15331516416, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.2.weight", "offset": 15356747776, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.2.weight", "offset": 15381979136, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.2.weight", "offset": 15407210496, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.3.weight", "offset": 15432441856, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.3.weight", "offset": 15457673216, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.3.weight", "offset": 15482904576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.4.weight", "offset": 15508135936, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.4.weight", "offset": 15533367296, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.4.weight", "offset": 15558598656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.5.weight", "offset": 15583830016, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.5.weight", "offset": 15609061376, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.5.weight", "offset": 15634292736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.6.weight", "offset": 15659524096, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.6.weight", "offset": 15684755456, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.6.weight", "offset": 15709986816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate.7.weight", "offset": 15735218176, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_down.7.weight", "offset": 15760449536, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_up.7.weight", "offset": 15785680896, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 15810912256, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 15810977792, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 15810994176, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 15811010560, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 15815467008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 15824904192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 15832113152, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.25.ffn_gate.0.weight", "offset": 15836569600, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.25.ffn_down.0.weight", "offset": 15861800960, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.0.weight", "offset": 15887032320, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate.1.weight", "offset": 15912263680, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.1.weight", "offset": 15937495040, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.1.weight", "offset": 15962726400, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate.2.weight", "offset": 15987957760, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.2.weight", "offset": 16013189120, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.2.weight", "offset": 16038420480, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate.3.weight", "offset": 16063651840, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.3.weight", "offset": 16088883200, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.3.weight", "offset": 16114114560, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 16139345920, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 16139411456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 16143867904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 16153305088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 16160514048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.25.ffn_gate.4.weight", "offset": 16164970496, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.25.ffn_down.4.weight", "offset": 16190201856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.4.weight", "offset": 16215433216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate.5.weight", "offset": 16240664576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.5.weight", "offset": 16265895936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.5.weight", "offset": 16291127296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_gate.6.weight", "offset": 16316358656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.6.weight", "offset": 16341590016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.6.weight", "offset": 16366821376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.25.ffn_up.7.we", "offset": 16392052736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_down.7.weight", "offset": 16417284096, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.25.ffn_up.7.weight", "offset": 16442515456, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 16467746816, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 16467763200, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.ffn_gate.0.weight", "offset": 16467779584, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.26.ffn_down.0.weight", "offset": 16493010944, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.0.weight", "offset": 16518242304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.1.weight", "offset": 16543473664, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.1.weight", "offset": 16568705024, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.1.weight", "offset": 16593936384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.2.weight", "offset": 16619167744, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.2.weight", "offset": 16644399104, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.2.weight", "offset": 16669630464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.3.weight", "offset": 16694861824, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.3.weight", "offset": 16720093184, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.3.weight", "offset": 16745324544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.4.weight", "offset": 16770555904, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.4.weight", "offset": 16795787264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.4.weight", "offset": 16821018624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.5.weight", "offset": 16846249984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.5.weight", "offset": 16871481344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.5.weight", "offset": 16896712704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.6.weight", "offset": 16921944064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.6.weight", "offset": 16947175424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.6.weight", "offset": 16972406784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate.7.weight", "offset": 16997638144, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_down.7.weight", "offset": 17022869504, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_up.7.weight", "offset": 17048100864, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 17073332224, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 17073397760, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 17073414144, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 17073430528, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 17077886976, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 17087324160, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 17094533120, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.27.ffn_gate.0.weight", "offset": 17098989568, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.27.ffn_down.0.weight", "offset": 17124220928, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.0.weight", "offset": 17149452288, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.1.weight", "offset": 17174683648, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.1.weight", "offset": 17199915008, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 17225146368, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 17225211904, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 17229668352, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 17239105536, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 17246314496, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.27.ffn_up.1.weight", "offset": 17250770944, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.27.ffn_gate.2.weight", "offset": 17276002304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.2.weight", "offset": 17301233664, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.2.weight", "offset": 17326465024, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.3.weight", "offset": 17351696384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.3.weight", "offset": 17376927744, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.3.weight", "offset": 17402159104, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.4.weight", "offset": 17427390464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.4.weight", "offset": 17452621824, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.4.weight", "offset": 17477853184, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.5.weight", "offset": 17503084544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.5.weight", "offset": 17528315904, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.5.weight", "offset": 17553547264, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.6.weight", "offset": 17578778624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.6.weight", "offset": 17604009984, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.6.weight", "offset": 17629241344, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_gate.7.weight", "offset": 17654472704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_down.7.weight", "offset": 17679704064, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.27.ffn_up.7.weight", "offset": 17704935424, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 17730166784, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 17730183168, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.ffn_gate.0.weight", "offset": 17730199552, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.28.ffn_down.0.weight", "offset": 17755430912, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.0.weight", "offset": 17780662272, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.1.weight", "offset": 17805893632, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.1.weight", "offset": 17831124992, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.1.weight", "offset": 17856356352, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.2.weight", "offset": 17881587712, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.2.weight", "offset": 17906819072, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.2.weight", "offset": 17932050432, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.3.weight", "offset": 17957281792, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.3.weight", "offset": 17982513152, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.3.weight", "offset": 18007744512, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.4.weight", "offset": 18032975872, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.4.weight", "offset": 18058207232, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.4.weight", "offset": 18083438592, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.5.weight", "offset": 18108669952, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.5.weight", "offset": 18133901312, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.5.weight", "offset": 18159132672, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.6.weight", "offset": 18184364032, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_down.6.weight", "offset": 18209595392, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_up.6.weight", "offset": 18234826752, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate.7.weight", "offset": 18260058112, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 18285289472, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 18285355008, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 18289811456, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 18299248640, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 18306457600, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.28.ffn_down.7.weight", "offset": 18310914048, "shape": [ 14336, 4096 ], "size": 4456448, "type": 11 }, { "name": "blk.28.ffn_up.7.weight", "offset": 18336145408, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 18361376768, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 18361393152, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.ffn_gate.0.weight", "offset": 18361409536, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.29.ffn_down.0.weight", "offset": 18386640896, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.0.weight", "offset": 18411872256, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.1.weight", "offset": 18437103616, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.1.weight", "offset": 18462334976, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.1.weight", "offset": 18487566336, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.2.weight", "offset": 18512797696, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.2.weight", "offset": 18538029056, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.2.weight", "offset": 18563260416, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.3.weight", "offset": 18588491776, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.3.weight", "offset": 18613723136, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.3.weight", "offset": 18638954496, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.4.weight", "offset": 18664185856, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.4.weight", "offset": 18689417216, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.4.weight", "offset": 18714648576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.5.weight", "offset": 18739879936, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.5.weight", "offset": 18765111296, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.5.weight", "offset": 18790342656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.6.weight", "offset": 18815574016, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.6.weight", "offset": 18840805376, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.6.weight", "offset": 18866036736, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate.7.weight", "offset": 18891268096, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_down.7.weight", "offset": 18916499456, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_up.7.weight", "offset": 18941730816, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 18966962176, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 18967027712, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 18967044096, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 18967060480, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 18971516928, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 18980954112, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 18988163072, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "blk.30.ffn_gate.0.weight", "offset": 18992619520, "shape": [ 4096, 14336 ], "size": 4456448, "type": 11 }, { "name": "blk.30.ffn_down.0.weight", "offset": 19017850880, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "\ufffd\ufffd1\ufffd\ufffd?\ufffds:\ufffdv\u0010uM\u0011\ufffd\ufffd\u0011\ufffdq\ufffd\ufffd", "offset": 19043082240, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.1.weight", "offset": 19068313600, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.1.weight", "offset": 19093544960, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.1.weight", "offset": 19118776320, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.2.weight", "offset": 19144007680, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.2.weight", "offset": 19169239040, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.2.weight", "offset": 19194470400, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.3.weight", "offset": 19219701760, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.3.weight", "offset": 19244933120, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.3.weight", "offset": 19270164480, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.4.weight", "offset": 19295395840, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.4.weight", "offset": 19320627200, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.4.weight", "offset": 19345858560, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 19371089920, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 19371155456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 19375611904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 19385049088, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 19392258048, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "output.weight", "offset": 19396714496, "shape": [ 4096, 32000 ], "size": 4456448, "type": 14 }, { "name": "blk.30.ffn_gate.5.weight", "offset": 19504234496, "shape": [ 4096, 14336 ], "size": 107520000, "type": 11 }, { "name": "blk.30.ffn_down.5.weight", "offset": 19529465856, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.5.weight", "offset": 19554697216, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.6.weight", "offset": 19579928576, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.6.weight", "offset": 19605159936, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.6.weight", "offset": 19630391296, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_gate.7.weight", "offset": 19655622656, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_down.7.weight", "offset": 19680854016, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.30.ffn_up.7.weight", "offset": 19706085376, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 19731316736, "shape": [ 4096 ], "size": 25231360, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 19731333120, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.ffn_gate.0.weight", "offset": 19731349504, "shape": [ 4096, 14336 ], "size": 16384, "type": 11 }, { "name": "blk.31.ffn_down.0.weight", "offset": 19756580864, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.0.weight", "offset": 19781812224, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.1.weight", "offset": 19807043584, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.1.weight", "offset": 19832274944, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.1.weight", "offset": 19857506304, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.2.weight", "offset": 19882737664, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.2.weight", "offset": 19907969024, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.2.weight", "offset": 19933200384, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.3.weight", "offset": 19958431744, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.3.weight", "offset": 19983663104, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.3.weight", "offset": 20008894464, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.4.weight", "offset": 20034125824, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.4.weight", "offset": 20059357184, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.4.weight", "offset": 20084588544, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.5.weight", "offset": 20109819904, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.5.weight", "offset": 20135051264, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.5.weight", "offset": 20160282624, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.6.weight", "offset": 20185513984, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.6.weight", "offset": 20210745344, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.6.weight", "offset": 20235976704, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate.7.weight", "offset": 20261208064, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_down.7.weight", "offset": 20286439424, "shape": [ 14336, 4096 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_up.7.weight", "offset": 20311670784, "shape": [ 4096, 14336 ], "size": 25231360, "type": 11 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 20336902144, "shape": [ 4096, 8 ], "size": 25231360, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 20336967680, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 20336984064, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 20337000448, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 20341456896, "shape": [ 4096, 4096 ], "size": 4456448, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 20350894080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 20358103040, "shape": [ 4096, 1024 ], "size": 7208960, "type": 8 }, { "name": "output_norm.weight", "offset": 20362559488, "shape": [ 4096 ], "size": 4456448, "type": 0 } ], "version": 3 }