codestral

{ "metadata": { "general.architecture": "llama", "general.file_type": 9, "general.name": "Codestral-22B-v0.1", "general.quantization_version": 2, "llama.attention.head_count": 48, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 56, "llama.context_length": 32768, "llama.embedding_length": 6144, "llama.feed_forward_length": 16384, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_space_prefix": true, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 22247282688, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 150994944, "shape": [ 6144 ], "size": 150994944, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 151019520, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 226516992, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 302014464, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 377511936, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 377536512, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 382255104, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.0.attn_q.weight", "offset": 410566656, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.0.attn_v.weight", "offset": 438878208, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 443596800, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 443621376, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 519118848, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 594616320, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 670113792, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 670138368, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 674856960, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.1.attn_q.weight", "offset": 703168512, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.1.attn_v.weight", "offset": 731480064, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 736198656, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 736223232, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 811720704, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 887218176, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 962715648, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "lk.17.attn_v.weight", "offset": 962740224, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 967458816, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.2.attn_q.weight", "offset": 995770368, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.2.attn_v.weight", "offset": 1024081920, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 1028800512, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1028825088, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 1104322560, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 1179820032, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1255317504, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1255342080, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 1260060672, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.3.attn_q.weight", "offset": 1288372224, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.3.attn_v.weight", "offset": 1316683776, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1321402368, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1321426944, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 1396924416, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1472421888, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1547919360, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1547943936, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 1552662528, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.4.attn_q.weight", "offset": 1580974080, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.4.attn_v.weight", "offset": 1609285632, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 1614004224, "shape": [ 6144, 16384 ], "size": 4718592, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1689501696, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.5.attn_k.weight", "offset": 1764999168, "shape": [ 6144, 1024 ], "size": 75497472, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 1769717760, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.5.attn_q.weight", "offset": 1798029312, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.5.attn_v.weight", "offset": 1826340864, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1831059456, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1831084032, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 1906581504, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1982078976, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 2057576448, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 2057601024, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 2062319616, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.10.attn_q.weight", "offset": 2090631168, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.10.attn_v.weight", "offset": 2118942720, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 2123661312, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2123685888, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 2199183360, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 2274680832, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 2350178304, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 2350202880, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 2354921472, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.11.attn_q.weight", "offset": 2383233024, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.11.attn_v.weight", "offset": 2411544576, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.12.attn_k.weight", "offset": 2416263168, "shape": [ 6144, 1024 ], "size": 4718592, "type": 7 }, { "name": "blk.12.attn_q.weight", "offset": 2420981760, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.12.attn_v.weight", "offset": 2449293312, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 2454011904, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 2454036480, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 2529533952, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 2529558528, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2529583104, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 2605080576, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 2680578048, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 2756075520, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2756100096, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 2760818688, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.6.attn_q.weight", "offset": 2789130240, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.6.attn_v.weight", "offset": 2817441792, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 2822160384, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2822184960, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 2897682432, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 2973179904, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 3048677376, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3048701952, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 3053420544, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.7.attn_q.weight", "offset": 3081732096, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.7.attn_v.weight", "offset": 3110043648, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 3114762240, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3114786816, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 3190284288, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 3265781760, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 3341279232, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3341303808, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 3346022400, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.8.attn_q.weight", "offset": 3374333952, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.8.attn_v.weight", "offset": 3402645504, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 3407364096, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3407388672, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 3482886144, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 3558383616, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 3633881088, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3633905664, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 3638624256, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.9.attn_q.weight", "offset": 3666935808, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.9.attn_v.weight", "offset": 3695247360, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 3699965952, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3699990528, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 3775488000, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 3850985472, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 3926482944, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3926507520, "shape": [ 6144, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 3954819072, "shape": [ 6144 ], "size": 28311552, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3954843648, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 4030341120, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 4105838592, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 4181336064, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 4181360640, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 4186079232, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.13.attn_q.weight", "offset": 4214390784, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.13.attn_v.weight", "offset": 4242702336, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 4247420928, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4247445504, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 4322942976, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 4398440448, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 4473937920, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 4473962496, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 4478681088, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.14.attn_q.weight", "offset": 4506992640, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.14.attn_v.weight", "offset": 4535304192, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 4540022784, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4540047360, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 4615544832, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 4691042304, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 4766539776, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 4766564352, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 4771282944, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.15.attn_q.weight", "offset": 4799594496, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.15.attn_v.weight", "offset": 4827906048, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 4832624640, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4832649216, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 4908146688, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 4983644160, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 5059141632, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 5059166208, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 5063884800, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.16.attn_q.weight", "offset": 5092196352, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.16.attn_v.weight", "offset": 5120507904, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 5125226496, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 5125251072, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 5200748544, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 5276246016, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 5351743488, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 5351768064, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 5356486656, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.17.attn_q.weight", "offset": 5384798208, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0007\u0000", "offset": 5413109760, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 5417828352, "shape": [ 6144, 16384 ], "size": 4718592, "type": 7 }, { "name": "blk.18.attn_k.weight", "offset": 5493325824, "shape": [ 6144, 1024 ], "size": 75497472, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 5498044416, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.18.attn_q.weight", "offset": 5526355968, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.18.attn_v.weight", "offset": 5554667520, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 5559386112, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5559410688, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 5634908160, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 5710405632, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 5710430208, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5710454784, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 5785952256, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 5861449728, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 5936947200, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 5936971776, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 5941690368, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.19.attn_q.weight", "offset": 5970001920, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.19.attn_v.weight", "offset": 5998313472, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 6003032064, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 6003056640, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 6078554112, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 6154051584, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 6229549056, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 6229573632, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 6234292224, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.20.attn_q.weight", "offset": 6262603776, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.20.attn_v.weight", "offset": 6290915328, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 6295633920, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 6295658496, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 6371155968, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 6446653440, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 6522150912, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 6522175488, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 6526894080, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.21.attn_q.weight", "offset": 6555205632, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.21.attn_v.weight", "offset": 6583517184, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 6588235776, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6588260352, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 6663757824, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 6739255296, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 6814752768, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 6814777344, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 6819495936, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.22.attn_q.weight", "offset": 6847807488, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.22.attn_v.weight", "offset": 6876119040, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 6880837632, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6880862208, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 6956359680, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 7031857152, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 7107354624, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 7107379200, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 7112097792, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.23.attn_q.weight", "offset": 7140409344, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.23.attn_v.weight", "offset": 7168720896, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 7173439488, "shape": [ 6144, 16384 ], "size": 4718592, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 7248936960, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.24.attn_k.weight", "offset": 7324434432, "shape": [ 6144, 1024 ], "size": 75497472, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 7329153024, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.24.attn_q.weight", "offset": 7357464576, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.24.attn_v.weight", "offset": 7385776128, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 7390494720, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 7390519296, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 7466016768, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 7466041344, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7466065920, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 7541563392, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 7617060864, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 7692558336, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 7692582912, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 7697301504, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.25.attn_q.weight", "offset": 7725613056, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.25.attn_v.weight", "offset": 7753924608, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 7758643200, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7758667776, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 7834165248, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 7909662720, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 7985160192, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 7985184768, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 7989903360, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.26.attn_q.weight", "offset": 8018214912, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.26.attn_v.weight", "offset": 8046526464, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 8051245056, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 8051269632, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 8126767104, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 8202264576, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 8277762048, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 8277786624, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 8282505216, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.27.attn_q.weight", "offset": 8310816768, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.27.attn_v.weight", "offset": 8339128320, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 8343846912, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 8343871488, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.28.ffn_gate.weight", "offset": 8419368960, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 8494866432, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 8570363904, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 8570388480, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 8575107072, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.28.attn_q.weight", "offset": 8603418624, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.28.attn_v.weight", "offset": 8631730176, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 8636448768, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 8636473344, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.29.ffn_gate.weight", "offset": 8711970816, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 8787468288, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 8862965760, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 8862990336, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 8867708928, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.29.attn_q.weight", "offset": 8896020480, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.29.attn_v.weight", "offset": 8924332032, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 8929050624, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 8929075200, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.30.ffn_gate.weight", "offset": 9004572672, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 9080070144, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 9155567616, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 9155592192, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 9160310784, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.30.attn_q.weight", "offset": 9188622336, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.30.attn_v.weight", "offset": 9216933888, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.31.attn_k.weight", "offset": 9221652480, "shape": [ 6144, 1024 ], "size": 4718592, "type": 7 }, { "name": "blk.31.attn_q.weight", "offset": 9226371072, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.31.attn_v.weight", "offset": 9254682624, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 9259401216, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 9259425792, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.31.ffn_gate.weight", "offset": 9334923264, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 9410420736, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 9485918208, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 9485942784, "shape": [ 6144, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 9514254336, "shape": [ 6144 ], "size": 28311552, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 9514278912, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.32.ffn_gate.weight", "offset": 9589776384, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 9665273856, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 9740771328, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 9740795904, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.32.attn_output.weight", "offset": 9745514496, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.32.attn_q.weight", "offset": 9773826048, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.32.attn_v.weight", "offset": 9802137600, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.33.attn_norm.weight", "offset": 9806856192, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "\u0000\u0000\u0000blk.48.ffn_up.weigh", "offset": 9806880768, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.33.ffn_gate.weight", "offset": 9882378240, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 9957875712, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 10033373184, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 10033397760, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.33.attn_output.weight", "offset": 10038116352, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.33.attn_q.weight", "offset": 10066427904, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.33.attn_v.weight", "offset": 10094739456, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 10099458048, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 10099482624, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.34.ffn_gate.weight", "offset": 10174980096, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 10250477568, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 10325975040, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 10325999616, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.34.attn_output.weight", "offset": 10330718208, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.34.attn_q.weight", "offset": 10359029760, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.34.attn_v.weight", "offset": 10387341312, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.35.attn_norm.weight", "offset": 10392059904, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 10392084480, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.35.ffn_gate.weight", "offset": 10467581952, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 10543079424, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 10618576896, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 10618601472, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 10623320064, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.35.attn_q.weight", "offset": 10651631616, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.35.attn_v.weight", "offset": 10679943168, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.36.attn_norm.weight", "offset": 10684661760, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 10684686336, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.36.ffn_gate.weight", "offset": 10760183808, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 10835681280, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 10911178752, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 10911203328, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.36.attn_output.weight", "offset": 10915921920, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.36.attn_q.weight", "offset": 10944233472, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.36.attn_v.weight", "offset": 10972545024, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.37.ffn_gate.weight", "offset": 10977263616, "shape": [ 6144, 16384 ], "size": 4718592, "type": 7 }, { "name": "blk.37.attn_k.weight", "offset": 11052761088, "shape": [ 6144, 1024 ], "size": 75497472, "type": 7 }, { "name": "blk.37.attn_output.weight", "offset": 11057479680, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.37.attn_q.weight", "offset": 11085791232, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.37.attn_v.weight", "offset": 11114102784, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 11118821376, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 11118845952, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 11194343424, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 11269840896, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.38.attn_norm.weight", "offset": 11269865472, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 11269890048, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.38.ffn_gate.weight", "offset": 11345387520, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 11420884992, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 11496382464, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 11496407040, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.38.attn_output.weight", "offset": 11501125632, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.38.attn_q.weight", "offset": 11529437184, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.38.attn_v.weight", "offset": 11557748736, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 11562467328, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 11562491904, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.39.ffn_gate.weight", "offset": 11637989376, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 11713486848, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 11788984320, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 11789008896, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.39.attn_output.weight", "offset": 11793727488, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.39.attn_q.weight", "offset": 11822039040, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.39.attn_v.weight", "offset": 11850350592, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.40.attn_norm.weight", "offset": 11855069184, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 11855093760, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.40.ffn_gate.weight", "offset": 11930591232, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.40.ffn_up.weight", "offset": 12006088704, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.40.ffn_norm.weight", "offset": 12081586176, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 12081610752, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.40.attn_output.weight", "offset": 12086329344, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.40.attn_q.weight", "offset": 12114640896, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.40.attn_v.weight", "offset": 12142952448, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.41.attn_norm.weight", "offset": 12147671040, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 12147695616, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.41.ffn_gate.weight", "offset": 12223193088, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.41.ffn_up.weight", "offset": 12298690560, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.41.ffn_norm.weight", "offset": 12374188032, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 12374212608, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.41.attn_output.weight", "offset": 12378931200, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.41.attn_q.weight", "offset": 12407242752, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.41.attn_v.weight", "offset": 12435554304, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.42.attn_norm.weight", "offset": 12440272896, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 12440297472, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.42.ffn_gate.weight", "offset": 12515794944, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.42.ffn_up.weight", "offset": 12591292416, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.42.ffn_norm.weight", "offset": 12666789888, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 12666814464, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.42.attn_output.weight", "offset": 12671533056, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.42.attn_q.weight", "offset": 12699844608, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.42.attn_v.weight", "offset": 12728156160, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.43.ffn_gate.weight", "offset": 12732874752, "shape": [ 6144, 16384 ], "size": 4718592, "type": 7 }, { "name": "blk.43.ffn_up.weight", "offset": 12808372224, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.43.attn_k.weight", "offset": 12883869696, "shape": [ 6144, 1024 ], "size": 75497472, "type": 7 }, { "name": "blk.43.attn_output.weight", "offset": 12888588288, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.43.attn_q.weight", "offset": 12916899840, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.43.attn_v.weight", "offset": 12945211392, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.43.attn_norm.weight", "offset": 12949929984, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 12949954560, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.43.ffn_norm.weight", "offset": 13025452032, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 13025476608, "shape": [ 6144 ], "size": 24576, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 13025501184, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.44.ffn_gate.weight", "offset": 13100998656, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.44.ffn_up.weight", "offset": 13176496128, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.44.ffn_norm.weight", "offset": 13251993600, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 13252018176, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.44.attn_output.weight", "offset": 13256736768, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.44.attn_q.weight", "offset": 13285048320, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.44.attn_v.weight", "offset": 13313359872, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.45.attn_norm.weight", "offset": 13318078464, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 13318103040, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.45.ffn_gate.weight", "offset": 13393600512, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.45.ffn_up.weight", "offset": 13469097984, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 13544595456, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 13544620032, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.45.attn_output.weight", "offset": 13549338624, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.45.attn_q.weight", "offset": 13577650176, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.45.attn_v.weight", "offset": 13605961728, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.46.attn_norm.weight", "offset": 13610680320, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 13610704896, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.46.ffn_gate.weight", "offset": 13686202368, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.46.ffn_up.weight", "offset": 13761699840, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.46.ffn_norm.weight", "offset": 13837197312, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 13837221888, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.46.attn_output.weight", "offset": 13841940480, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.46.attn_q.weight", "offset": 13870252032, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.46.attn_v.weight", "offset": 13898563584, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.47.attn_norm.weight", "offset": 13903282176, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 13903306752, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.47.ffn_gate.weight", "offset": 13978804224, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.47.ffn_up.weight", "offset": 14054301696, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.47.ffn_norm.weight", "offset": 14129799168, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 14129823744, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.47.attn_output.weight", "offset": 14134542336, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.47.attn_q.weight", "offset": 14162853888, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.47.attn_v.weight", "offset": 14191165440, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.48.attn_norm.weight", "offset": 14195884032, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 14195908608, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.48.ffn_gate.weight", "offset": 14271406080, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 14346903552, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.48.ffn_norm.weight", "offset": 14422401024, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 14422425600, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.48.attn_output.weight", "offset": 14427144192, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.48.attn_q.weight", "offset": 14455455744, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.48.attn_v.weight", "offset": 14483767296, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.49.attn_norm.weight", "offset": 14488485888, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 14488510464, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.49.ffn_gate.weight", "offset": 14564007936, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.49.ffn_up.weight", "offset": 14639505408, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.49.ffn_norm.weight", "offset": 14715002880, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 14715027456, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.49.attn_output.weight", "offset": 14719746048, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.49.attn_q.weight", "offset": 14748057600, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.49.attn_v.weight", "offset": 14776369152, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.50.attn_k.weight", "offset": 14781087744, "shape": [ 6144, 1024 ], "size": 4718592, "type": 7 }, { "name": "blk.50.attn_q.weight", "offset": 14785806336, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.50.attn_v.weight", "offset": 14814117888, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "output.weight", "offset": 14818836480, "shape": [ 6144, 32768 ], "size": 4718592, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 14983987200, "shape": [ 6144 ], "size": 165150720, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 14984011776, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.50.ffn_gate.weight", "offset": 15059509248, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.50.ffn_up.weight", "offset": 15135006720, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.50.ffn_norm.weight", "offset": 15210504192, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.50.attn_output.weight", "offset": 15210528768, "shape": [ 6144, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.51.attn_norm.weight", "offset": 15238840320, "shape": [ 6144 ], "size": 28311552, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 15238864896, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.51.ffn_gate.weight", "offset": 15314362368, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.51.ffn_up.weight", "offset": 15389859840, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.51.ffn_norm.weight", "offset": 15465357312, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.51.attn_k.weight", "offset": 15465381888, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.51.attn_output.weight", "offset": 15470100480, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.51.attn_q.weight", "offset": 15498412032, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.51.attn_v.weight", "offset": 15526723584, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.52.attn_norm.weight", "offset": 15531442176, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 15531466752, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.52.ffn_gate.weight", "offset": 15606964224, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.52.ffn_up.weight", "offset": 15682461696, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.52.ffn_norm.weight", "offset": 15757959168, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 15757983744, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.52.attn_output.weight", "offset": 15762702336, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.52.attn_q.weight", "offset": 15791013888, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.52.attn_v.weight", "offset": 15819325440, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.53.attn_norm.weight", "offset": 15824044032, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 15824068608, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.53.ffn_gate.weight", "offset": 15899566080, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.53.ffn_up.weight", "offset": 15975063552, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.53.ffn_norm.weight", "offset": 16050561024, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 16050585600, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.53.attn_output.weight", "offset": 16055304192, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.53.attn_q.weight", "offset": 16083615744, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.53.attn_v.weight", "offset": 16111927296, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.54.attn_norm.weight", "offset": 16116645888, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 16116670464, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.54.ffn_gate.weight", "offset": 16192167936, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.54.ffn_up.weight", "offset": 16267665408, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.54.ffn_norm.weight", "offset": 16343162880, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 16343187456, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.54.attn_output.weight", "offset": 16347906048, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.54.attn_q.weight", "offset": 16376217600, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.54.attn_v.weight", "offset": 16404529152, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "blk.55.attn_norm.weight", "offset": 16409247744, "shape": [ 6144 ], "size": 4718592, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 16409272320, "shape": [ 16384, 6144 ], "size": 24576, "type": 7 }, { "name": "blk.55.ffn_gate.weight", "offset": 16484769792, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.55.ffn_up.weight", "offset": 16560267264, "shape": [ 6144, 16384 ], "size": 75497472, "type": 7 }, { "name": "blk.55.ffn_norm.weight", "offset": 16635764736, "shape": [ 6144 ], "size": 75497472, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 16635789312, "shape": [ 6144, 1024 ], "size": 24576, "type": 7 }, { "name": "blk.55.attn_output.weight", "offset": 16640507904, "shape": [ 6144, 6144 ], "size": 4718592, "type": 7 }, { "name": "blk.55.attn_q.weight", "offset": 16668819456, "shape": [ 6144, 6144 ], "size": 28311552, "type": 7 }, { "name": "blk.55.attn_v.weight", "offset": 16697131008, "shape": [ 6144, 1024 ], "size": 28311552, "type": 7 }, { "name": "output_norm.weight", "offset": 16701849600, "shape": [ 6144 ], "size": 4718592, "type": 0 } ], "version": 3 }