mixtral

{ "metadata": { "general.architecture": "llama", "general.file_type": 9, "general.name": "mistralai", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 32768, "llama.embedding_length": 4096, "llama.expert_count": 8, "llama.expert_used_count": 2, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.merges": "... (58980 values)", "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 46702792704, "tensors": [ { "name": "blk.0.ffn_gate.0.weight", "offset": 98304000, "shape": [ 4096, 14336 ], "size": 98304000, "type": 7 }, { "name": "blk.0.ffn_down.0.weight", "offset": 142344192, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.0.weight", "offset": 186384384, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.1.weight", "offset": 230424576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.1.weight", "offset": 274464768, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.1.weight", "offset": 318504960, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.2.weight", "offset": 362545152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.2.weight", "offset": 406585344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.2.weight", "offset": 450625536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.3.weight", "offset": 494665728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.3.weight", "offset": 538705920, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.3.weight", "offset": 582746112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.4.weight", "offset": 626786304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.4.weight", "offset": 670826496, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.4.weight", "offset": 714866688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.5.weight", "offset": 758906880, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.5.weight", "offset": 802947072, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.5.weight", "offset": 846987264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.6.weight", "offset": 891027456, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.6.weight", "offset": 935067648, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.6.weight", "offset": 979107840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate.7.weight", "offset": 1023148032, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_down.7.weight", "offset": 1067188224, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_up.7.weight", "offset": 1111228416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.0.ffn_gate_inp.weight", "offset": 1155268608, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 1155334144, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1155350528, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1155366912, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 1159823360, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.0.attn_q.weight", "offset": 1172406272, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_v.weight", "offset": 1184989184, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.1.ffn_gate.0.weight", "offset": 1189445632, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.1.ffn_down.0.weight", "offset": 1233485824, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.0.weight", "offset": 1277526016, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.1.weight", "offset": 1321566208, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.1.weight", "offset": 1365606400, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.1.weight", "offset": 1409646592, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.2.weight", "offset": 1453686784, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.2.weight", "offset": 1497726976, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.2.weight", "offset": 1541767168, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.3.weight", "offset": 1585807360, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.3.weight", "offset": 1629847552, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.3.weight", "offset": 1673887744, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.4.weight", "offset": 1717927936, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.4.weight", "offset": 1761968128, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1806008320, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.1.attn_k.weight", "offset": 1806073856, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1810530304, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.1.attn_q.weight", "offset": 1823113216, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.1.attn_v.weight", "offset": 1835696128, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.1.ffn_up.4.weight", "offset": 1840152576, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.1.ffn_gate.5.weight", "offset": 1884192768, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.5.weight", "offset": 1928232960, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.5.weight", "offset": 1972273152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.6.weight", "offset": 2016313344, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.6.weight", "offset": 2060353536, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.6.weight", "offset": 2104393728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_gate.7.weight", "offset": 2148433920, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_down.7.weight", "offset": 2192474112, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.1.ffn_up.7.weight", "offset": 2236514304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 2280554496, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 2280570880, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.ffn_gate.0.weight", "offset": 2280587264, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.2.ffn_down.0.weight", "offset": 2324627456, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.0.weight", "offset": 2368667648, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.1.weight", "offset": 2412707840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.1.weight", "offset": 2456748032, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.1.weight", "offset": 2500788224, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.2.weight", "offset": 2544828416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.2.weight", "offset": 2588868608, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.2.weight", "offset": 2632908800, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.3.weight", "offset": 2676948992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.3.weight", "offset": 2720989184, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.3.weight", "offset": 2765029376, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.4.weight", "offset": 2809069568, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.4.weight", "offset": 2853109760, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.4.weight", "offset": 2897149952, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.5.weight", "offset": 2941190144, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.5.weight", "offset": 2985230336, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.5.weight", "offset": 3029270528, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.6.weight", "offset": 3073310720, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.6.weight", "offset": 3117350912, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.6.weight", "offset": 3161391104, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate.7.weight", "offset": 3205431296, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_down.7.weight", "offset": 3249471488, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_up.7.weight", "offset": 3293511680, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3337551872, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 3337617408, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 3337633792, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 3337650176, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 3342106624, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.2.attn_q.weight", "offset": 3354689536, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_v.weight", "offset": 3367272448, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.3.ffn_gate.0.weight", "offset": 3371728896, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.3.ffn_down.0.weight", "offset": 3415769088, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.0.weight", "offset": 3459809280, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.1.weight", "offset": 3503849472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.1.weight", "offset": 3547889664, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.1.weight", "offset": 3591929856, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.2.weight", "offset": 3635970048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 3680010240, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.3.attn_k.weight", "offset": 3680075776, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 3684532224, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.3.attn_q.weight", "offset": 3697115136, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_v.weight", "offset": 3709698048, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.3.ffn_down.2.weight", "offset": 3714154496, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.3.ffn_up.2.weight", "offset": 3758194688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.3.weight", "offset": 3802234880, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.3.weight", "offset": 3846275072, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.3.weight", "offset": 3890315264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.4.weight", "offset": 3934355456, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.4.weight", "offset": 3978395648, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.4.weight", "offset": 4022435840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.5.weight", "offset": 4066476032, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.5.weight", "offset": 4110516224, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.5.weight", "offset": 4154556416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.6.weight", "offset": 4198596608, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.6.weight", "offset": 4242636800, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.6.weight", "offset": 4286676992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_gate.7.weight", "offset": 4330717184, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_down.7.weight", "offset": 4374757376, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.3.ffn_up.7.weight", "offset": 4418797568, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 4462837760, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 4462854144, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.ffn_gate.0.weight", "offset": 4462870528, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.4.ffn_down.0.weight", "offset": 4506910720, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.0.weight", "offset": 4550950912, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.1.weight", "offset": 4594991104, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.1.weight", "offset": 4639031296, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.1.weight", "offset": 4683071488, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.2.weight", "offset": 4727111680, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.2.weight", "offset": 4771151872, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.2.weight", "offset": 4815192064, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.3.weight", "offset": 4859232256, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.3.weight", "offset": 4903272448, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.3.weight", "offset": 4947312640, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.4.weight", "offset": 4991352832, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.4.weight", "offset": 5035393024, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.4.weight", "offset": 5079433216, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.5.weight", "offset": 5123473408, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.5.weight", "offset": 5167513600, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.5.weight", "offset": 5211553792, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.6.weight", "offset": 5255593984, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.6.weight", "offset": 5299634176, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.6.weight", "offset": 5343674368, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate.7.weight", "offset": 5387714560, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_down.7.weight", "offset": 5431754752, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_up.7.weight", "offset": 5475794944, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 5519835136, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 5519900672, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 5519917056, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 5519933440, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 5524389888, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.4.attn_q.weight", "offset": 5536972800, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_v.weight", "offset": 5549555712, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 5554012160, "shape": [ 4096, 8 ], "size": 4456448, "type": 1 }, { "name": "blk.5.attn_k.weight", "offset": 5554077696, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 5558534144, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.5.attn_q.weight", "offset": 5571117056, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_v.weight", "offset": 5583699968, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.5.ffn_gate.0.weight", "offset": 5588156416, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.5.ffn_down.0.weight", "offset": 5632196608, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.0.weight", "offset": 5676236800, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.1.weight", "offset": 5720276992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.1.weight", "offset": 5764317184, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.1.weight", "offset": 5808357376, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.2.weight", "offset": 5852397568, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.2.weight", "offset": 5896437760, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.2.weight", "offset": 5940477952, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.3.weight", "offset": 5984518144, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.3.weight", "offset": 6028558336, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.3.weight", "offset": 6072598528, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.4.weight", "offset": 6116638720, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.4.weight", "offset": 6160678912, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.4.weight", "offset": 6204719104, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.5.weight", "offset": 6248759296, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.5.weight", "offset": 6292799488, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.5.weight", "offset": 6336839680, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.6.weight", "offset": 6380879872, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.6.weight", "offset": 6424920064, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.6.weight", "offset": 6468960256, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_gate.7.weight", "offset": 6513000448, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_down.7.weight", "offset": 6557040640, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.5.ffn_up.7.weight", "offset": 6601080832, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 6645121024, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 6645137408, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.ffn_gate.0.weight", "offset": 6645153792, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.6.ffn_down.0.weight", "offset": 6689193984, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.0.weight", "offset": 6733234176, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.1.weight", "offset": 6777274368, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.1.weight", "offset": 6821314560, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.1.weight", "offset": 6865354752, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.2.weight", "offset": 6909394944, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.2.weight", "offset": 6953435136, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.2.weight", "offset": 6997475328, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.3.weight", "offset": 7041515520, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.3.weight", "offset": 7085555712, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.3.weight", "offset": 7129595904, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.4.weight", "offset": 7173636096, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.4.weight", "offset": 7217676288, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.4.weight", "offset": 7261716480, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.5.weight", "offset": 7305756672, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.5.weight", "offset": 7349796864, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 7393837056, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 7393902592, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 7398359040, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.6.attn_q.weight", "offset": 7410941952, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_v.weight", "offset": 7423524864, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.6.ffn_up.5.weight", "offset": 7427981312, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.6.ffn_gate.6.weight", "offset": 7472021504, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.6.weight", "offset": 7516061696, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.6.weight", "offset": 7560101888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_gate.7.weight", "offset": 7604142080, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_down.7.weight", "offset": 7648182272, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.6.ffn_up.7.weight", "offset": 7692222464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 7736262656, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 7736279040, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.ffn_gate.0.weight", "offset": 7736295424, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.7.ffn_down.0.weight", "offset": 7780335616, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.0.weight", "offset": 7824375808, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.1.weight", "offset": 7868416000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.1.weight", "offset": 7912456192, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.1.weight", "offset": 7956496384, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.2.weight", "offset": 8000536576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.2.weight", "offset": 8044576768, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.2.weight", "offset": 8088616960, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.3.weight", "offset": 8132657152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.3.weight", "offset": 8176697344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.3.weight", "offset": 8220737536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.4.weight", "offset": 8264777728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.4.weight", "offset": 8308817920, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.4.weight", "offset": 8352858112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.5.weight", "offset": 8396898304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.5.weight", "offset": 8440938496, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.5.weight", "offset": 8484978688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.6.weight", "offset": 8529018880, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.6.weight", "offset": 8573059072, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.6.weight", "offset": 8617099264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate.7.weight", "offset": 8661139456, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_down.7.weight", "offset": 8705179648, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_up.7.weight", "offset": 8749219840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 8793260032, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 8793325568, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 8793341952, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 8793358336, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 8797814784, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.7.attn_q.weight", "offset": 8810397696, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_v.weight", "offset": 8822980608, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.8.ffn_gate.0.weight", "offset": 8827437056, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.8.ffn_down.0.weight", "offset": 8871477248, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.0.weight", "offset": 8915517440, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.1.weight", "offset": 8959557632, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.1.weight", "offset": 9003597824, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.1.weight", "offset": 9047638016, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.2.weight", "offset": 9091678208, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.2.weight", "offset": 9135718400, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.2.weight", "offset": 9179758592, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.3.weight", "offset": 9223798784, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 9267838976, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.8.attn_k.weight", "offset": 9267904512, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 9272360960, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.8.attn_q.weight", "offset": 9284943872, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_v.weight", "offset": 9297526784, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.10.ffn_gate.0.weight", "offset": 9301983232, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.10.ffn_down.0.weight", "offset": 9346023424, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.0.weight", "offset": 9390063616, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 9434103808, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 9434169344, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 9438625792, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.10.attn_q.weight", "offset": 9451208704, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_v.weight", "offset": 9463791616, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.8.ffn_down.3.weight", "offset": 9468248064, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.8.ffn_up.3.weight", "offset": 9512288256, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.4.weight", "offset": 9556328448, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.4.weight", "offset": 9600368640, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.4.weight", "offset": 9644408832, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.5.weight", "offset": 9688449024, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.5.weight", "offset": 9732489216, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.5.weight", "offset": 9776529408, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.6.weight", "offset": 9820569600, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.6.weight", "offset": 9864609792, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.6.weight", "offset": 9908649984, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_gate.7.weight", "offset": 9952690176, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_down.7.weight", "offset": 9996730368, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.8.ffn_up.7.weight", "offset": 10040770560, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 10084810752, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 10084827136, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.ffn_gate.0.weight", "offset": 10084843520, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.9.ffn_down.0.weight", "offset": 10128883712, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.0.weight", "offset": 10172923904, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.1.weight", "offset": 10216964096, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.1.weight", "offset": 10261004288, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.1.weight", "offset": 10305044480, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.2.weight", "offset": 10349084672, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.2.weight", "offset": 10393124864, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.2.weight", "offset": 10437165056, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.3.weight", "offset": 10481205248, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.3.weight", "offset": 10525245440, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.3.weight", "offset": 10569285632, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.4.weight", "offset": 10613325824, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.4.weight", "offset": 10657366016, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.4.weight", "offset": 10701406208, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.5.weight", "offset": 10745446400, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.5.weight", "offset": 10789486592, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.5.weight", "offset": 10833526784, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.6.weight", "offset": 10877566976, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.6.weight", "offset": 10921607168, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.6.weight", "offset": 10965647360, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate.7.weight", "offset": 11009687552, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_down.7.weight", "offset": 11053727744, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_up.7.weight", "offset": 11097767936, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 11141808128, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 11141873664, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 11141890048, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 11141906432, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 11146362880, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.9.attn_q.weight", "offset": 11158945792, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_v.weight", "offset": 11171528704, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.10.ffn_gate.1.weight", "offset": 11175985152, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.10.ffn_down.1.weight", "offset": 11220025344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.1.weight", "offset": 11264065536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.2.weight", "offset": 11308105728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.2.weight", "offset": 11352145920, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.2.weight", "offset": 11396186112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.3.weight", "offset": 11440226304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.3.weight", "offset": 11484266496, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.3.weight", "offset": 11528306688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.4.weight", "offset": 11572346880, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.4.weight", "offset": 11616387072, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.4.weight", "offset": 11660427264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.5.weight", "offset": 11704467456, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.5.weight", "offset": 11748507648, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.5.weight", "offset": 11792547840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.6.weight", "offset": 11836588032, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.6.weight", "offset": 11880628224, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.6.weight", "offset": 11924668416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_gate.7.weight", "offset": 11968708608, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_down.7.weight", "offset": 12012748800, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.10.ffn_up.7.weight", "offset": 12056788992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 12100829184, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 12100845568, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.ffn_gate.0.weight", "offset": 12100861952, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.11.ffn_down.0.weight", "offset": 12144902144, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.0.weight", "offset": 12188942336, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.1.weight", "offset": 12232982528, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.1.weight", "offset": 12277022720, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.1.weight", "offset": 12321062912, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.2.weight", "offset": 12365103104, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.2.weight", "offset": 12409143296, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.2.weight", "offset": 12453183488, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.3.weight", "offset": 12497223680, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.3.weight", "offset": 12541263872, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.3.weight", "offset": 12585304064, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.4.weight", "offset": 12629344256, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.4.weight", "offset": 12673384448, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.4.weight", "offset": 12717424640, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.5.weight", "offset": 12761464832, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.5.weight", "offset": 12805505024, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.5.weight", "offset": 12849545216, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate.6.weight", "offset": 12893585408, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.6.weight", "offset": 12937625600, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 12981665792, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.11.attn_k.weight", "offset": 12981731328, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 12986187776, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.11.attn_q.weight", "offset": 12998770688, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_v.weight", "offset": 13011353600, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.11.ffn_up.6.weight", "offset": 13015810048, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.11.ffn_gate.7.weight", "offset": 13059850240, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_down.7.weight", "offset": 13103890432, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.11.ffn_up.7.weight", "offset": 13147930624, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 13191970816, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 13191987200, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.ffn_gate.0.weight", "offset": 13192003584, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.12.ffn_down.0.weight", "offset": 13236043776, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.0.weight", "offset": 13280083968, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.1.weight", "offset": 13324124160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.1.weight", "offset": 13368164352, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.1.weight", "offset": 13412204544, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.2.weight", "offset": 13456244736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.2.weight", "offset": 13500284928, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.2.weight", "offset": 13544325120, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.3.weight", "offset": 13588365312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.3.weight", "offset": 13632405504, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.3.weight", "offset": 13676445696, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.4.weight", "offset": 13720485888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.4.weight", "offset": 13764526080, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.4.weight", "offset": 13808566272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.5.weight", "offset": 13852606464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.5.weight", "offset": 13896646656, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.5.weight", "offset": 13940686848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.6.weight", "offset": 13984727040, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.6.weight", "offset": 14028767232, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.6.weight", "offset": 14072807424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate.7.weight", "offset": 14116847616, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_down.7.weight", "offset": 14160887808, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_up.7.weight", "offset": 14204928000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 14248968192, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 14249033728, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 14249050112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 14249066496, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 14253522944, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.12.attn_q.weight", "offset": 14266105856, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_v.weight", "offset": 14278688768, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.13.ffn_gate.0.weight", "offset": 14283145216, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.13.ffn_down.0.weight", "offset": 14327185408, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.0.weight", "offset": 14371225600, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.1.weight", "offset": 14415265792, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.1.weight", "offset": 14459305984, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.1.weight", "offset": 14503346176, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.2.weight", "offset": 14547386368, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.2.weight", "offset": 14591426560, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.2.weight", "offset": 14635466752, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.3.weight", "offset": 14679506944, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.3.weight", "offset": 14723547136, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.3.weight", "offset": 14767587328, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.4.weight", "offset": 14811627520, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 14855667712, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.13.attn_k.weight", "offset": 14855733248, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 14860189696, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.13.attn_q.weight", "offset": 14872772608, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_v.weight", "offset": 14885355520, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.13.ffn_down.4.weight", "offset": 14889811968, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.13.ffn_up.4.weight", "offset": 14933852160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.5.weight", "offset": 14977892352, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.5.weight", "offset": 15021932544, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.5.weight", "offset": 15065972736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.6.weight", "offset": 15110012928, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.6.weight", "offset": 15154053120, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.6.weight", "offset": 15198093312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_gate.7.weight", "offset": 15242133504, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_down.7.weight", "offset": 15286173696, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.13.ffn_up.7.weight", "offset": 15330213888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 15374254080, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 15374270464, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.ffn_gate.0.weight", "offset": 15374286848, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.14.ffn_down.0.weight", "offset": 15418327040, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.0.weight", "offset": 15462367232, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.1.weight", "offset": 15506407424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.1.weight", "offset": 15550447616, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.1.weight", "offset": 15594487808, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.2.weight", "offset": 15638528000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.2.weight", "offset": 15682568192, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.2.weight", "offset": 15726608384, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.3.weight", "offset": 15770648576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.3.weight", "offset": 15814688768, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.3.weight", "offset": 15858728960, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.4.weight", "offset": 15902769152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.4.weight", "offset": 15946809344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.4.weight", "offset": 15990849536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.5.weight", "offset": 16034889728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.5.weight", "offset": 16078929920, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.5.weight", "offset": 16122970112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.6.weight", "offset": 16167010304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.6.weight", "offset": 16211050496, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.6.weight", "offset": 16255090688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate.7.weight", "offset": 16299130880, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_down.7.weight", "offset": 16343171072, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_up.7.weight", "offset": 16387211264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 16431251456, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 16431316992, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 16431333376, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 16431349760, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 16435806208, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.14.attn_q.weight", "offset": 16448389120, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_v.weight", "offset": 16460972032, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.15.ffn_gate.0.weight", "offset": 16465428480, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.15.ffn_down.0.weight", "offset": 16509468672, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.0.weight", "offset": 16553508864, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.1.weight", "offset": 16597549056, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.1.weight", "offset": 16641589248, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.1.weight", "offset": 16685629440, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 16729669632, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 16729735168, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 16734191616, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.15.attn_q.weight", "offset": 16746774528, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_v.weight", "offset": 16759357440, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.15.ffn_gate.2.weight", "offset": 16763813888, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.15.ffn_down.2.weight", "offset": 16807854080, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.2.weight", "offset": 16851894272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.3.weight", "offset": 16895934464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.3.weight", "offset": 16939974656, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.3.weight", "offset": 16984014848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.4.weight", "offset": 17028055040, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.4.weight", "offset": 17072095232, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.4.weight", "offset": 17116135424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.5.weight", "offset": 17160175616, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.5.weight", "offset": 17204215808, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.5.weight", "offset": 17248256000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.6.weight", "offset": 17292296192, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.6.weight", "offset": 17336336384, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.6.weight", "offset": 17380376576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_gate.7.weight", "offset": 17424416768, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_down.7.weight", "offset": 17468456960, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.15.ffn_up.7.weight", "offset": 17512497152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 17556537344, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 17556553728, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.ffn_gate.0.weight", "offset": 17556570112, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.16.ffn_down.0.weight", "offset": 17600610304, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.0.weight", "offset": 17644650496, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.1.weight", "offset": 17688690688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.1.weight", "offset": 17732730880, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.1.weight", "offset": 17776771072, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.2.weight", "offset": 17820811264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.2.weight", "offset": 17864851456, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.2.weight", "offset": 17908891648, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.3.weight", "offset": 17952931840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.3.weight", "offset": 17996972032, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.3.weight", "offset": 18041012224, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.4.weight", "offset": 18085052416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.4.weight", "offset": 18129092608, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.4.weight", "offset": 18173132800, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.5.weight", "offset": 18217172992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.5.weight", "offset": 18261213184, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.5.weight", "offset": 18305253376, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.6.weight", "offset": 18349293568, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.6.weight", "offset": 18393333760, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_up.6.weight", "offset": 18437373952, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate.7.weight", "offset": 18481414144, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_down.7.weight", "offset": 18525454336, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 18569494528, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.16.attn_k.weight", "offset": 18569560064, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 18574016512, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.16.attn_q.weight", "offset": 18586599424, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_v.weight", "offset": 18599182336, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.16.ffn_up.7.weight", "offset": 18603638784, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 18647678976, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 18647695360, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.ffn_gate.0.weight", "offset": 18647711744, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.17.ffn_down.0.weight", "offset": 18691751936, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.0.weight", "offset": 18735792128, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.1.weight", "offset": 18779832320, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.1.weight", "offset": 18823872512, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.1.weight", "offset": 18867912704, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.2.weight", "offset": 18911952896, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.2.weight", "offset": 18955993088, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.2.weight", "offset": 19000033280, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.3.weight", "offset": 19044073472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.3.weight", "offset": 19088113664, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.3.weight", "offset": 19132153856, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.4.weight", "offset": 19176194048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.4.weight", "offset": 19220234240, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.4.weight", "offset": 19264274432, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.5.weight", "offset": 19308314624, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.5.weight", "offset": 19352354816, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.5.weight", "offset": 19396395008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.6.weight", "offset": 19440435200, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.6.weight", "offset": 19484475392, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.6.weight", "offset": 19528515584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate.7.weight", "offset": 19572555776, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_down.7.weight", "offset": 19616595968, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_up.7.weight", "offset": 19660636160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 19704676352, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 19704741888, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 19704758272, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 19704774656, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 19709231104, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.17.attn_q.weight", "offset": 19721814016, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_v.weight", "offset": 19734396928, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.18.ffn_gate.0.weight", "offset": 19738853376, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.18.ffn_down.0.weight", "offset": 19782893568, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.0.weight", "offset": 19826933760, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.1.weight", "offset": 19870973952, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.1.weight", "offset": 19915014144, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.1.weight", "offset": 19959054336, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.2.weight", "offset": 20003094528, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.2.weight", "offset": 20047134720, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.2.weight", "offset": 20091174912, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.3.weight", "offset": 20135215104, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.3.weight", "offset": 20179255296, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.3.weight", "offset": 20223295488, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.4.weight", "offset": 20267335680, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.4.weight", "offset": 20311375872, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.4.weight", "offset": 20355416064, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.5.weight", "offset": 20399456256, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 20443496448, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.18.attn_k.weight", "offset": 20443561984, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 20448018432, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.18.attn_q.weight", "offset": 20460601344, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_v.weight", "offset": 20473184256, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.18.ffn_down.5.weight", "offset": 20477640704, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.18.ffn_up.5.weight", "offset": 20521680896, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.6.weight", "offset": 20565721088, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.6.weight", "offset": 20609761280, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.6.weight", "offset": 20653801472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_gate.7.weight", "offset": 20697841664, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_down.7.weight", "offset": 20741881856, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.18.ffn_up.7.weight", "offset": 20785922048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 20829962240, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 20829978624, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.ffn_gate.0.weight", "offset": 20829995008, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.19.ffn_down.0.weight", "offset": 20874035200, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.0.weight", "offset": 20918075392, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.1.weight", "offset": 20962115584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.1.weight", "offset": 21006155776, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.1.weight", "offset": 21050195968, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.2.weight", "offset": 21094236160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.2.weight", "offset": 21138276352, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.2.weight", "offset": 21182316544, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.3.weight", "offset": 21226356736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.3.weight", "offset": 21270396928, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.3.weight", "offset": 21314437120, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.4.weight", "offset": 21358477312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.4.weight", "offset": 21402517504, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.4.weight", "offset": 21446557696, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.5.weight", "offset": 21490597888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.5.weight", "offset": 21534638080, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.5.weight", "offset": 21578678272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.6.weight", "offset": 21622718464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.6.weight", "offset": 21666758656, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.6.weight", "offset": 21710798848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate.7.weight", "offset": 21754839040, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_down.7.weight", "offset": 21798879232, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_up.7.weight", "offset": 21842919424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 21886959616, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 21887025152, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 21887041536, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 21887057920, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 21891514368, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.19.attn_q.weight", "offset": 21904097280, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_v.weight", "offset": 21916680192, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.20.ffn_gate.0.weight", "offset": 21921136640, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.20.ffn_down.0.weight", "offset": 21965176832, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.0.weight", "offset": 22009217024, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.1.weight", "offset": 22053257216, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.1.weight", "offset": 22097297408, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.1.weight", "offset": 22141337600, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.2.weight", "offset": 22185377792, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.2.weight", "offset": 22229417984, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.2.weight", "offset": 22273458176, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 22317498368, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.20.attn_k.weight", "offset": 22317563904, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 22322020352, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.20.attn_q.weight", "offset": 22334603264, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_v.weight", "offset": 22347186176, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.20.ffn_gate.3.weight", "offset": 22351642624, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.20.ffn_down.3.weight", "offset": 22395682816, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.3.weight", "offset": 22439723008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.4.weight", "offset": 22483763200, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.4.weight", "offset": 22527803392, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.4.weight", "offset": 22571843584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.5.weight", "offset": 22615883776, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.5.weight", "offset": 22659923968, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.5.weight", "offset": 22703964160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.6.weight", "offset": 22748004352, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.6.weight", "offset": 22792044544, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.6.weight", "offset": 22836084736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_gate.7.weight", "offset": 22880124928, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_down.7.weight", "offset": 22924165120, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.20.ffn_up.7.weight", "offset": 22968205312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 23012245504, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 23012261888, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.ffn_gate.0.weight", "offset": 23012278272, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.21.ffn_down.0.weight", "offset": 23056318464, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.0.weight", "offset": 23100358656, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.1.weight", "offset": 23144398848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.1.weight", "offset": 23188439040, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.1.weight", "offset": 23232479232, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.2.weight", "offset": 23276519424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.2.weight", "offset": 23320559616, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.2.weight", "offset": 23364599808, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.3.weight", "offset": 23408640000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.3.weight", "offset": 23452680192, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.3.weight", "offset": 23496720384, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.4.weight", "offset": 23540760576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.4.weight", "offset": 23584800768, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.4.weight", "offset": 23628840960, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.5.weight", "offset": 23672881152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.5.weight", "offset": 23716921344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.5.weight", "offset": 23760961536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.6.weight", "offset": 23805001728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.6.weight", "offset": 23849041920, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.6.weight", "offset": 23893082112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate.7.weight", "offset": 23937122304, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_down.7.weight", "offset": 23981162496, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_up.7.weight", "offset": 24025202688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 24069242880, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 24069308416, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 24069324800, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 24069341184, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 24073797632, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.21.attn_q.weight", "offset": 24086380544, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_v.weight", "offset": 24098963456, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.22.ffn_gate.0.weight", "offset": 24103419904, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.22.ffn_down.0.weight", "offset": 24147460096, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 24191500288, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.22.attn_k.weight", "offset": 24191565824, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 24196022272, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.22.attn_q.weight", "offset": 24208605184, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_v.weight", "offset": 24221188096, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.22.ffn_up.0.weight", "offset": 24225644544, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.22.ffn_gate.1.weight", "offset": 24269684736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.1.weight", "offset": 24313724928, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.1.weight", "offset": 24357765120, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.2.weight", "offset": 24401805312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.2.weight", "offset": 24445845504, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.2.weight", "offset": 24489885696, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.3.weight", "offset": 24533925888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.3.weight", "offset": 24577966080, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.3.weight", "offset": 24622006272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.4.weight", "offset": 24666046464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.4.weight", "offset": 24710086656, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.4.weight", "offset": 24754126848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.5.weight", "offset": 24798167040, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.5.weight", "offset": 24842207232, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.5.weight", "offset": 24886247424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.6.weight", "offset": 24930287616, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.6.weight", "offset": 24974327808, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.6.weight", "offset": 25018368000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_gate.7.weight", "offset": 25062408192, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_down.7.weight", "offset": 25106448384, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.22.ffn_up.7.weight", "offset": 25150488576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 25194528768, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 25194545152, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.ffn_gate.0.weight", "offset": 25194561536, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.23.ffn_down.0.weight", "offset": 25238601728, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.0.weight", "offset": 25282641920, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.1.weight", "offset": 25326682112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.1.weight", "offset": 25370722304, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.1.weight", "offset": 25414762496, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.2.weight", "offset": 25458802688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.2.weight", "offset": 25502842880, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.2.weight", "offset": 25546883072, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.3.weight", "offset": 25590923264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.3.weight", "offset": 25634963456, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.3.weight", "offset": 25679003648, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.4.weight", "offset": 25723043840, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.4.weight", "offset": 25767084032, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.4.weight", "offset": 25811124224, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.5.weight", "offset": 25855164416, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.5.weight", "offset": 25899204608, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.5.weight", "offset": 25943244800, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.6.weight", "offset": 25987284992, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 26031325184, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.23.attn_k.weight", "offset": 26031390720, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 26035847168, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.23.attn_q.weight", "offset": 26048430080, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_v.weight", "offset": 26061012992, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.23.ffn_down.6.weight", "offset": 26065469440, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.23.ffn_up.6.weight", "offset": 26109509632, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_gate.7.weight", "offset": 26153549824, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_down.7.weight", "offset": 26197590016, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.23.ffn_up.7.weight", "offset": 26241630208, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 26285670400, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 26285686784, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.ffn_gate.0.weight", "offset": 26285703168, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.24.ffn_down.0.weight", "offset": 26329743360, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.0.weight", "offset": 26373783552, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.1.weight", "offset": 26417823744, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.1.weight", "offset": 26461863936, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.1.weight", "offset": 26505904128, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.2.weight", "offset": 26549944320, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.2.weight", "offset": 26593984512, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.2.weight", "offset": 26638024704, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.3.weight", "offset": 26682064896, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.3.weight", "offset": 26726105088, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.3.weight", "offset": 26770145280, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.4.weight", "offset": 26814185472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.4.weight", "offset": 26858225664, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.4.weight", "offset": 26902265856, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.5.weight", "offset": 26946306048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.5.weight", "offset": 26990346240, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.5.weight", "offset": 27034386432, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.6.weight", "offset": 27078426624, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.6.weight", "offset": 27122466816, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.6.weight", "offset": 27166507008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate.7.weight", "offset": 27210547200, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_down.7.weight", "offset": 27254587392, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_up.7.weight", "offset": 27298627584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 27342667776, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 27342733312, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 27342749696, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 27342766080, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 27347222528, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.24.attn_q.weight", "offset": 27359805440, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_v.weight", "offset": 27372388352, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.25.ffn_gate.0.weight", "offset": 27376844800, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.25.ffn_down.0.weight", "offset": 27420884992, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.0.weight", "offset": 27464925184, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.1.weight", "offset": 27508965376, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.1.weight", "offset": 27553005568, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.1.weight", "offset": 27597045760, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.2.weight", "offset": 27641085952, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.2.weight", "offset": 27685126144, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.2.weight", "offset": 27729166336, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.3.weight", "offset": 27773206528, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.3.weight", "offset": 27817246720, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.3.weight", "offset": 27861286912, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 27905327104, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.25.attn_k.weight", "offset": 27905392640, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 27909849088, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.25.attn_q.weight", "offset": 27922432000, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_v.weight", "offset": 27935014912, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.25.ffn_gate.4.weight", "offset": 27939471360, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.25.ffn_down.4.weight", "offset": 27983511552, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.4.weight", "offset": 28027551744, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.5.weight", "offset": 28071591936, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.5.weight", "offset": 28115632128, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.5.weight", "offset": 28159672320, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.6.weight", "offset": 28203712512, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.6.weight", "offset": 28247752704, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.6.weight", "offset": 28291792896, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_gate.7.weight", "offset": 28335833088, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_down.7.weight", "offset": 28379873280, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.25.ffn_up.7.weight", "offset": 28423913472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 28467953664, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 28467970048, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.ffn_gate.0.weight", "offset": 28467986432, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.26.ffn_down.0.weight", "offset": 28512026624, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.0.weight", "offset": 28556066816, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.1.weight", "offset": 28600107008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.1.weight", "offset": 28644147200, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.1.weight", "offset": 28688187392, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.2.weight", "offset": 28732227584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.2.weight", "offset": 28776267776, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.2.weight", "offset": 28820307968, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.3.weight", "offset": 28864348160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.3.weight", "offset": 28908388352, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.3.weight", "offset": 28952428544, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.4.weight", "offset": 28996468736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.4.weight", "offset": 29040508928, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.4.weight", "offset": 29084549120, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.5.weight", "offset": 29128589312, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.5.weight", "offset": 29172629504, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.5.weight", "offset": 29216669696, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.6.weight", "offset": 29260709888, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.6.weight", "offset": 29304750080, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.6.weight", "offset": 29348790272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate.7.weight", "offset": 29392830464, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_down.7.weight", "offset": 29436870656, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_up.7.weight", "offset": 29480910848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 29524951040, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 29525016576, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 29525032960, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 29525049344, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 29529505792, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.26.attn_q.weight", "offset": 29542088704, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_v.weight", "offset": 29554671616, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.27.ffn_gate.0.weight", "offset": 29559128064, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.27.ffn_down.0.weight", "offset": 29603168256, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.0.weight", "offset": 29647208448, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.1.weight", "offset": 29691248640, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.1.weight", "offset": 29735288832, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 29779329024, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.27.attn_k.weight", "offset": 29779394560, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 29783851008, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.27.attn_q.weight", "offset": 29796433920, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_v.weight", "offset": 29809016832, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.27.ffn_up.1.weight", "offset": 29813473280, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.27.ffn_gate.2.weight", "offset": 29857513472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.2.weight", "offset": 29901553664, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.2.weight", "offset": 29945593856, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.3.weight", "offset": 29989634048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.3.weight", "offset": 30033674240, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.3.weight", "offset": 30077714432, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.4.weight", "offset": 30121754624, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.4.weight", "offset": 30165794816, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.4.weight", "offset": 30209835008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.5.weight", "offset": 30253875200, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.5.weight", "offset": 30297915392, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.5.weight", "offset": 30341955584, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.6.weight", "offset": 30385995776, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.6.weight", "offset": 30430035968, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.6.weight", "offset": 30474076160, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_gate.7.weight", "offset": 30518116352, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_down.7.weight", "offset": 30562156544, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.27.ffn_up.7.weight", "offset": 30606196736, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 30650236928, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 30650253312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.ffn_gate.0.weight", "offset": 30650269696, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.28.ffn_down.0.weight", "offset": 30694309888, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.0.weight", "offset": 30738350080, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.1.weight", "offset": 30782390272, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.1.weight", "offset": 30826430464, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.1.weight", "offset": 30870470656, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.2.weight", "offset": 30914510848, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.2.weight", "offset": 30958551040, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.2.weight", "offset": 31002591232, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.3.weight", "offset": 31046631424, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.3.weight", "offset": 31090671616, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.3.weight", "offset": 31134711808, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.4.weight", "offset": 31178752000, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.4.weight", "offset": 31222792192, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.4.weight", "offset": 31266832384, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.5.weight", "offset": 31310872576, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.5.weight", "offset": 31354912768, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.5.weight", "offset": 31398952960, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.6.weight", "offset": 31442993152, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_down.6.weight", "offset": 31487033344, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_up.6.weight", "offset": 31531073536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate.7.weight", "offset": 31575113728, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 31619153920, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 31619219456, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 31623675904, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.28.attn_q.weight", "offset": 31636258816, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_v.weight", "offset": 31648841728, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.28.ffn_down.7.weight", "offset": 31653298176, "shape": [ 14336, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.28.ffn_up.7.weight", "offset": 31697338368, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 31741378560, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 31741394944, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.ffn_gate.0.weight", "offset": 31741411328, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.29.ffn_down.0.weight", "offset": 31785451520, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.0.weight", "offset": 31829491712, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.1.weight", "offset": 31873531904, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.1.weight", "offset": 31917572096, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.1.weight", "offset": 31961612288, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.2.weight", "offset": 32005652480, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.2.weight", "offset": 32049692672, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.2.weight", "offset": 32093732864, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.3.weight", "offset": 32137773056, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.3.weight", "offset": 32181813248, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.3.weight", "offset": 32225853440, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.4.weight", "offset": 32269893632, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.4.weight", "offset": 32313933824, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.4.weight", "offset": 32357974016, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.5.weight", "offset": 32402014208, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.5.weight", "offset": 32446054400, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.5.weight", "offset": 32490094592, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.6.weight", "offset": 32534134784, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.6.weight", "offset": 32578174976, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.6.weight", "offset": 32622215168, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate.7.weight", "offset": 32666255360, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_down.7.weight", "offset": 32710295552, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_up.7.weight", "offset": 32754335744, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 32798375936, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 32798441472, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 32798457856, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 32798474240, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 32802930688, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.29.attn_q.weight", "offset": 32815513600, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_v.weight", "offset": 32828096512, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "blk.30.ffn_gate.0.weight", "offset": 32832552960, "shape": [ 4096, 14336 ], "size": 4456448, "type": 7 }, { "name": "blk.30.ffn_down.0.weight", "offset": 32876593152, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.0.weight", "offset": 32920633344, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.1.weight", "offset": 32964673536, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.1.weight", "offset": 33008713728, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.1.weight", "offset": 33052753920, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.2.weight", "offset": 33096794112, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.2.weight", "offset": 33140834304, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.2.weight", "offset": 33184874496, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.3.weight", "offset": 33228914688, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.3.weight", "offset": 33272954880, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.3.weight", "offset": 33316995072, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.4.weight", "offset": 33361035264, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.4.weight", "offset": 33405075456, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.4.weight", "offset": 33449115648, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 33493155840, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.30.attn_k.weight", "offset": 33493221376, "shape": [ 4096, 1024 ], "size": 65536, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 33497677824, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.30.attn_q.weight", "offset": 33510260736, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_v.weight", "offset": 33522843648, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "output.weight", "offset": 33527300096, "shape": [ 4096, 32000 ], "size": 4456448, "type": 14 }, { "name": "blk.30.ffn_gate.5.weight", "offset": 33634820096, "shape": [ 4096, 14336 ], "size": 107520000, "type": 7 }, { "name": "blk.30.ffn_down.5.weight", "offset": 33678860288, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.5.weight", "offset": 33722900480, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.6.weight", "offset": 33766940672, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.6.weight", "offset": 33810980864, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.6.weight", "offset": 33855021056, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_gate.7.weight", "offset": 33899061248, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_down.7.weight", "offset": 33943101440, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.30.ffn_up.7.weight", "offset": 33987141632, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 34031181824, "shape": [ 4096 ], "size": 44040192, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 34031198208, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.ffn_gate.0.weight", "offset": 34031214592, "shape": [ 4096, 14336 ], "size": 16384, "type": 7 }, { "name": "blk.31.ffn_down.0.weight", "offset": 34075254784, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.0.weight", "offset": 34119294976, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.1.weight", "offset": 34163335168, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.1.weight", "offset": 34207375360, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.1.weight", "offset": 34251415552, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.2.weight", "offset": 34295455744, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.2.weight", "offset": 34339495936, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.2.weight", "offset": 34383536128, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.3.weight", "offset": 34427576320, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.3.weight", "offset": 34471616512, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.3.weight", "offset": 34515656704, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.4.weight", "offset": 34559696896, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.4.weight", "offset": 34603737088, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.4.weight", "offset": 34647777280, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.5.weight", "offset": 34691817472, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.5.weight", "offset": 34735857664, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.5.weight", "offset": 34779897856, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.6.weight", "offset": 34823938048, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.6.weight", "offset": 34867978240, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.6.weight", "offset": 34912018432, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate.7.weight", "offset": 34956058624, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_down.7.weight", "offset": 35000098816, "shape": [ 14336, 4096 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_up.7.weight", "offset": 35044139008, "shape": [ 4096, 14336 ], "size": 44040192, "type": 7 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 35088179200, "shape": [ 4096, 8 ], "size": 44040192, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 35088244736, "shape": [ 4096 ], "size": 65536, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 35088261120, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 35088277504, "shape": [ 4096, 1024 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 35092733952, "shape": [ 4096, 4096 ], "size": 4456448, "type": 7 }, { "name": "blk.31.attn_q.weight", "offset": 35105316864, "shape": [ 4096, 4096 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_v.weight", "offset": 35117899776, "shape": [ 4096, 1024 ], "size": 12582912, "type": 8 }, { "name": "output_norm.weight", "offset": 35122356224, "shape": [ 4096 ], "size": 4456448, "type": 0 } ], "version": 3 }