gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 1, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 339757056, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 679495680, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.0.post_attention_norm.weight", "offset": 1019234304, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 1019252736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 1019271168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 1019289600, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 1038163968, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.0.attn_q.weight", "offset": 1075912704, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.0.attn_v.weight", "offset": 1113661440, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 1132535808, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1132554240, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.1.ffn_gate.weight", "offset": 1472292864, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 1812031488, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.1.post_attention_norm.weight", "offset": 2151770112, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 2151788544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 2151806976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 2151825408, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 2170699776, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.1.attn_q.weight", "offset": 2208448512, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.1.attn_v.weight", "offset": 2246197248, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.2.attn_k.weight", "offset": 2265071616, "shape": [ 4608, 2048 ], "size": 18874368, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 2283945984, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.2.attn_q.weight", "offset": 2321694720, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.2.attn_v.weight", "offset": 2359443456, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 2378317824, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2378336256, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.2.ffn_gate.weight", "offset": 2718074880, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 3057813504, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.2.post_attention_norm.weight", "offset": 3397552128, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 3397570560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 3397588992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 3397607424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 3397625856, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.3.ffn_gate.weight", "offset": 3737364480, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 4077103104, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.3.post_attention_norm.weight", "offset": 4416841728, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 4416860160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 4416878592, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 4416897024, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 4435771392, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.3.attn_q.weight", "offset": 4473520128, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.3.attn_v.weight", "offset": 4511268864, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 4530143232, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 4530161664, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.4.ffn_gate.weight", "offset": 4869900288, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 5209638912, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.4.post_attention_norm.weight", "offset": 5549377536, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 5549395968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 5549414400, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 5549432832, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 5568307200, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.4.attn_q.weight", "offset": 5606055936, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.4.attn_v.weight", "offset": 5643804672, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 5662679040, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 5662697472, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.5.ffn_gate.weight", "offset": 6002436096, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 6342174720, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.17.attn_q.weight\u0002\u0000\u0000\u0000\u0000\u0012", "offset": 6681913344, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 6681931776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 6681950208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 6681968640, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 6700843008, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.5.attn_q.weight", "offset": 6738591744, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.5.attn_v.weight", "offset": 6776340480, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.6.ffn_gate.weight", "offset": 6795214848, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.6.attn_k.weight", "offset": 7134953472, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 7153827840, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.6.attn_q.weight", "offset": 7191576576, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.6.attn_v.weight", "offset": 7229325312, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 7248199680, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 7248218112, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 7587956736, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.6.post_attention_norm.weight", "offset": 7927695360, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 7927713792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 7927732224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 7927750656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 7927769088, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.7.ffn_gate.weight", "offset": 8267507712, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 8607246336, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.7.post_attention_norm.weight", "offset": 8946984960, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 8947003392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 8947021824, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 8947040256, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 8965914624, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.7.attn_q.weight", "offset": 9003663360, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.7.attn_v.weight", "offset": 9041412096, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 9060286464, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 9060304896, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.8.ffn_gate.weight", "offset": 9400043520, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 9739782144, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.8.post_attention_norm.weight", "offset": 10079520768, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 10079539200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 10079557632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 10079576064, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 10098450432, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.8.attn_q.weight", "offset": 10136199168, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.8.attn_v.weight", "offset": 10173947904, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 10192822272, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 10192840704, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.9.ffn_gate.weight", "offset": 10532579328, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 10872317952, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.9.post_attention_norm.weight", "offset": 11212056576, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 11212075008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 11212093440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 11212111872, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 11230986240, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.9.attn_q.weight", "offset": 11268734976, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.9.attn_v.weight", "offset": 11306483712, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.10.ffn_gate.weight", "offset": 11325358080, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 11665096704, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.10.attn_k.weight", "offset": 12004835328, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 12023709696, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.10.attn_q.weight", "offset": 12061458432, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.10.attn_v.weight", "offset": 12099207168, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 12118081536, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 12118099968, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.10.post_attention_norm.weight", "offset": 12457838592, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 12457857024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 12457875456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 12457893888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 12457912320, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.11.ffn_gate.weight", "offset": 12797650944, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 13137389568, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.11.post_attention_norm.weight", "offset": 13477128192, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 13477146624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 13477165056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 13477183488, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 13496057856, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.11.attn_q.weight", "offset": 13533806592, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.11.attn_v.weight", "offset": 13571555328, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 13590429696, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 13590448128, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.12.ffn_gate.weight", "offset": 13930186752, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 14269925376, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.12.post_attention_norm.weight", "offset": 14609664000, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 14609682432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 14609700864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 14609719296, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 14628593664, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.12.attn_q.weight", "offset": 14666342400, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.12.attn_v.weight", "offset": 14704091136, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 14722965504, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 14722983936, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.13.ffn_gate.weight", "offset": 15062722560, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 15402461184, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.13.post_attention_norm.weight", "offset": 15742199808, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 15742218240, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 15742236672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 15742255104, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 15761129472, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.13.attn_q.weight", "offset": 15798878208, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.13.attn_v.weight", "offset": 15836626944, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 15855501312, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 15855519744, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.14.ffn_gate.weight", "offset": 16195258368, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 16534996992, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.14.post_attention_norm.weight", "offset": 16874735616, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 16874754048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 16874772480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 16874790912, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 16893665280, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.14.attn_q.weight", "offset": 16931414016, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.14.attn_v.weight", "offset": 16969162752, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.15.attn_k.weight", "offset": 16988037120, "shape": [ 4608, 2048 ], "size": 18874368, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 17006911488, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.15.attn_q.weight", "offset": 17044660224, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.15.attn_v.weight", "offset": 17082408960, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 17101283328, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 17101301760, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.15.ffn_gate.weight", "offset": 17441040384, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 17780779008, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.15.post_attention_norm.weight", "offset": 18120517632, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 18120536064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 18120554496, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 18120572928, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 18120591360, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.16.ffn_gate.weight", "offset": 18460329984, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 18800068608, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.16.post_attention_norm.weight", "offset": 19139807232, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 19139825664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 19139844096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 19139862528, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 19158736896, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.16.attn_q.weight", "offset": 19196485632, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.16.attn_v.weight", "offset": 19234234368, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 19253108736, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 19253127168, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.17.ffn_gate.weight", "offset": 19592865792, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 19932604416, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.17.post_attention_norm.weight", "offset": 20272343040, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 20272361472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 20272379904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 20272398336, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 20291272704, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "ffn_gate.weight\u0002\u0000\u0000\u0000\u0000", "offset": 20329021440, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.17.attn_v.weight", "offset": 20366770176, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 20385644544, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 20385662976, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.18.ffn_gate.weight", "offset": 20725401600, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 21065140224, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.18.post_attention_norm.weight", "offset": 21404878848, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 21404897280, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 21404915712, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 21404934144, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 21423808512, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.18.attn_q.weight", "offset": 21461557248, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.18.attn_v.weight", "offset": 21499305984, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.19.ffn_gate.weight", "offset": 21518180352, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.19.attn_k.weight", "offset": 21857918976, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 21876793344, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.19.attn_q.weight", "offset": 21914542080, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.19.attn_v.weight", "offset": 21952290816, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 21971165184, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 21971183616, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 22310922240, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.19.post_attention_norm.weight", "offset": 22650660864, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 22650679296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 22650697728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 22650716160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 22650734592, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.20.ffn_gate.weight", "offset": 22990473216, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 23330211840, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.20.post_attention_norm.weight", "offset": 23669950464, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 23669968896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 23669987328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 23670005760, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 23688880128, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.20.attn_q.weight", "offset": 23726628864, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.20.attn_v.weight", "offset": 23764377600, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 23783251968, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 23783270400, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.21.ffn_gate.weight", "offset": 24123009024, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 24462747648, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.21.post_attention_norm.weight", "offset": 24802486272, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 24802504704, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 24802523136, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 24802541568, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 24821415936, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.21.attn_q.weight", "offset": 24859164672, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.21.attn_v.weight", "offset": 24896913408, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 24915787776, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 24915806208, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.22.ffn_gate.weight", "offset": 25255544832, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 25595283456, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.22.post_attention_norm.weight", "offset": 25935022080, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 25935040512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 25935058944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 25935077376, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 25953951744, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.22.attn_q.weight", "offset": 25991700480, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.22.attn_v.weight", "offset": 26029449216, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 26048323584, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.23.ffn_gate.weight", "offset": 26388062208, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.23.attn_k.weight", "offset": 26727800832, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 26746675200, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.23.attn_q.weight", "offset": 26784423936, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.23.attn_v.weight", "offset": 26822172672, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 26841047040, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 26841065472, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.23.post_attention_norm.weight", "offset": 27180804096, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 27180822528, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 27180840960, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 27180859392, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 27180877824, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.24.ffn_gate.weight", "offset": 27520616448, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 27860355072, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.24.post_attention_norm.weight", "offset": 28200093696, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 28200112128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 28200130560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 28200148992, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 28219023360, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.24.attn_q.weight", "offset": 28256772096, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.24.attn_v.weight", "offset": 28294520832, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 28313395200, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 28313413632, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.25.ffn_gate.weight", "offset": 28653152256, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 28992890880, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.25.post_attention_norm.weight", "offset": 29332629504, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 29332647936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 29332666368, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 29332684800, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 29351559168, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.25.attn_q.weight", "offset": 29389307904, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.25.attn_v.weight", "offset": 29427056640, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 29445931008, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 29445949440, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.26.ffn_gate.weight", "offset": 29785688064, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 30125426688, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.26.post_attention_norm.weight", "offset": 30465165312, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 30465183744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 30465202176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 30465220608, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 30484094976, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.26.attn_q.weight", "offset": 30521843712, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.26.attn_v.weight", "offset": 30559592448, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 30578466816, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 30578485248, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.27.ffn_gate.weight", "offset": 30918223872, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 31257962496, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.27.post_attention_norm.weight", "offset": 31597701120, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 31597719552, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 31597737984, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 31597756416, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 31616630784, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.27.attn_q.weight", "offset": 31654379520, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.27.attn_v.weight", "offset": 31692128256, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.28.attn_k.weight", "offset": 31711002624, "shape": [ 4608, 2048 ], "size": 18874368, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 31729876992, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.28.attn_q.weight", "offset": 31767625728, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.28.attn_v.weight", "offset": 31805374464, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 31824248832, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 31824267264, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.28.ffn_gate.weight", "offset": 32164005888, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 32503744512, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.28.post_attention_norm.weight", "offset": 32843483136, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 32843501568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 32843520000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 32843538432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 32843556864, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.29.ffn_gate.weight", "offset": 33183295488, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 33523034112, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.29.post_attention_norm.weight", "offset": 33862772736, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 33862791168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 33862809600, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 33862828032, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 33881702400, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.29.attn_q.weight", "offset": 33919451136, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.29.attn_v.weight", "offset": 33957199872, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 33976074240, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 33976092672, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "lk.42.ffn_norm.weight\u0001", "offset": 34315831296, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 34655569920, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.30.post_attention_norm.weight", "offset": 34995308544, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 34995326976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 34995345408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 34995363840, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 35014238208, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.30.attn_q.weight", "offset": 35051986944, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.30.attn_v.weight", "offset": 35089735680, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 35108610048, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 35108628480, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.31.ffn_gate.weight", "offset": 35448367104, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 35788105728, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.31.post_attention_norm.weight", "offset": 36127844352, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 36127862784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 36127881216, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 36127899648, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.31.attn_output.weight", "offset": 36146774016, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.31.attn_q.weight", "offset": 36184522752, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.31.attn_v.weight", "offset": 36222271488, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.32.ffn_gate.weight", "offset": 36241145856, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.32.attn_k.weight", "offset": 36580884480, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 36599758848, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.32.attn_q.weight", "offset": 36637507584, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.32.attn_v.weight", "offset": 36675256320, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 36694130688, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 36694149120, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 37033887744, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.32.post_attention_norm.weight", "offset": 37373626368, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 37373644800, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 37373663232, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 37373681664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 37373700096, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.33.ffn_gate.weight", "offset": 37713438720, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 38053177344, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.33.post_attention_norm.weight", "offset": 38392915968, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 38392934400, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 38392952832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 38392971264, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 38411845632, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.33.attn_q.weight", "offset": 38449594368, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.33.attn_v.weight", "offset": 38487343104, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 38506217472, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 38506235904, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.34.ffn_gate.weight", "offset": 38845974528, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 39185713152, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.34.post_attention_norm.weight", "offset": 39525451776, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 39525470208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 39525488640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 39525507072, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 39544381440, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.34.attn_q.weight", "offset": 39582130176, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.34.attn_v.weight", "offset": 39619878912, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 39638753280, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 39638771712, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.35.ffn_gate.weight", "offset": 39978510336, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 40318248960, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.35.post_attention_norm.weight", "offset": 40657987584, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 40658006016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 40658024448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 40658042880, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 40676917248, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.35.attn_q.weight", "offset": 40714665984, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.35.attn_v.weight", "offset": 40752414720, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.36.ffn_gate.weight", "offset": 40771289088, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 41111027712, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.36.attn_k.weight", "offset": 41450766336, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 41469640704, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.36.attn_q.weight", "offset": 41507389440, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.36.attn_v.weight", "offset": 41545138176, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.36.attn_norm.weight", "offset": 41564012544, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 41564030976, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.36.post_attention_norm.weight", "offset": 41903769600, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 41903788032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 41903806464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 41903824896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 41903843328, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.37.ffn_gate.weight", "offset": 42243581952, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 42583320576, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.37.post_attention_norm.weight", "offset": 42923059200, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 42923077632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 42923096064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 42923114496, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 42941988864, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.37.attn_q.weight", "offset": 42979737600, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.37.attn_v.weight", "offset": 43017486336, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 43036360704, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 43036379136, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.38.ffn_gate.weight", "offset": 43376117760, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 43715856384, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.38.post_attention_norm.weight", "offset": 44055595008, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 44055613440, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 44055631872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 44055650304, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 44074524672, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.38.attn_q.weight", "offset": 44112273408, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.38.attn_v.weight", "offset": 44150022144, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 44168896512, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 44168914944, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.39.ffn_gate.weight", "offset": 44508653568, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 44848392192, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.39.post_attention_norm.weight", "offset": 45188130816, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 45188149248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 45188167680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 45188186112, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 45207060480, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.39.attn_q.weight", "offset": 45244809216, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.39.attn_v.weight", "offset": 45282557952, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.40.ffn_down.weight", "offset": 45301432320, "shape": [ 36864, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 45641170944, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 45641189376, "shape": [ 4608, 36864 ], "size": 18432, "type": 1 }, { "name": "blk.40.ffn_up.weight", "offset": 45980928000, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.40.post_attention_norm.weight", "offset": 46320666624, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 46320685056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 46320703488, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 46320721920, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 46339596288, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.40.attn_q.weight", "offset": 46377345024, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.40.attn_v.weight", "offset": 46415093760, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.41.attn_k.weight", "offset": 46433968128, "shape": [ 4608, 2048 ], "size": 18874368, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 46452842496, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.41.attn_q.weight", "offset": 46490591232, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.41.attn_v.weight", "offset": 46528339968, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 46547214336, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 46547232768, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.41.ffn_gate.weight", "offset": 46886971392, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.41.ffn_up.weight", "offset": 47226710016, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.41.post_attention_norm.weight", "offset": 47566448640, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 47566467072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 47566485504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 47566503936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 47566522368, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.42.ffn_gate.weight", "offset": 47906260992, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.42.ffn_up.weight", "offset": 48245999616, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.42.post_attention_norm.weight", "offset": 48585738240, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 48585756672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 48585775104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 48585793536, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.42.attn_output.weight", "offset": 48604667904, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.42.attn_q.weight", "offset": 48642416640, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.42.attn_v.weight", "offset": 48680165376, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.43.attn_norm.weight", "offset": 48699039744, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 48699058176, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.43.ffn_gate.weight", "offset": 49038796800, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.43.ffn_up.weight", "offset": 49378535424, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.43.post_attention_norm.weight", "offset": 49718274048, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 49718292480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 49718310912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 49718329344, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.43.attn_output.weight", "offset": 49737203712, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.43.attn_q.weight", "offset": 49774952448, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.43.attn_v.weight", "offset": 49812701184, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.44.attn_norm.weight", "offset": 49831575552, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 49831593984, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.44.ffn_gate.weight", "offset": 50171332608, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.44.ffn_up.weight", "offset": 50511071232, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.44.post_attention_norm.weight", "offset": 50850809856, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 50850828288, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 50850846720, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 50850865152, "shape": [ 4608, 2048 ], "size": 18432, "type": 1 }, { "name": "blk.44.attn_output.weight", "offset": 50869739520, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.44.attn_q.weight", "offset": 50907488256, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.44.attn_v.weight", "offset": 50945236992, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.45.ffn_gate.weight", "offset": 50964111360, "shape": [ 4608, 36864 ], "size": 18874368, "type": 1 }, { "name": "blk.45.attn_k.weight", "offset": 51303849984, "shape": [ 4608, 2048 ], "size": 339738624, "type": 1 }, { "name": "blk.45.attn_output.weight", "offset": 51322724352, "shape": [ 4096, 4608 ], "size": 18874368, "type": 1 }, { "name": "blk.45.attn_q.weight", "offset": 51360473088, "shape": [ 4608, 4096 ], "size": 37748736, "type": 1 }, { "name": "blk.45.attn_v.weight", "offset": 51398221824, "shape": [ 4608, 2048 ], "size": 37748736, "type": 1 }, { "name": "blk.45.attn_norm.weight", "offset": 51417096192, "shape": [ 4608 ], "size": 18874368, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 51417114624, "shape": [ 36864, 4608 ], "size": 18432, "type": 1 }, { "name": "blk.45.ffn_up.weight", "offset": 51756853248, "shape": [ 4608, 36864 ], "size": 339738624, "type": 1 }, { "name": "blk.45.post_attention_norm.weight", "offset": 52096591872, "shape": [ 4608 ], "size": 339738624, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 52096610304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 52096628736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 52096647168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 52096665600, "shape": [ 4608, 256000 ], "size": 18432, "type": 1 } ], "version": 3 }