gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 13, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 116803584, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 189794304, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.0.post_attention_norm.weight", "offset": 262785024, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 262803456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 262821888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 262840320, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 266895360, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 279871488, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 287981568, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 294469632, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 294488064, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 411273216, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 484263936, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.1.post_attention_norm.weight", "offset": 557254656, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 557273088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 557291520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 557309952, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 561364992, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 574341120, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 582451200, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.2.attn_k.weight", "offset": 588939264, "shape": [ 4608, 2048 ], "size": 6488064, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 592994304, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 605970432, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 614080512, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 620568576, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 620587008, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 737372160, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 810362880, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.2.post_attention_norm.weight", "offset": 883353600, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 883372032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 883390464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 883408896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 883427328, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 1000212480, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 1073203200, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.3.post_attention_norm.weight", "offset": 1146193920, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 1146212352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 1146230784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1146249216, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 1150304256, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1163280384, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 1171390464, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1177878528, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1177896960, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 1294682112, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 1367672832, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1440663552, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1440681984, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1440700416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1440718848, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1444773888, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1457750016, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 1465860096, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1472348160, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1472366592, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 1589151744, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 1662142464, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.5.post_attention_norm.weight", "offset": 1735133184, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 1735151616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1735170048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1735188480, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1739243520, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1752219648, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 1760329728, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 1766817792, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.6.attn_k.weight", "offset": 1839808512, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1843863552, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 1856839680, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 1864949760, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 1871437824, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1871456256, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1988241408, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.6.post_attention_norm.weight", "offset": 2061232128, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 2061250560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 2061268992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2061287424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2061305856, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 2178091008, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 2251081728, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.7.post_attention_norm.weight", "offset": 2324072448, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 2324090880, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 2324109312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2324127744, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 2328182784, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 2341158912, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 2349268992, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2355757056, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2355775488, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 2472560640, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 2545551360, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.8.post_attention_norm.weight", "offset": 2618542080, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 2618560512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 2618578944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2618597376, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 2622652416, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 2635628544, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 2643738624, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 2650226688, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2650245120, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 2767030272, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2840020992, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.9.post_attention_norm.weight", "offset": 2913011712, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 2913030144, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2913048576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2913067008, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 2917122048, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 2930098176, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 2938208256, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 2944696320, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 3017687040, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 3090677760, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 3094732800, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 3107708928, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 3115819008, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 3122307072, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 3122325504, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.10.post_attention_norm.weight", "offset": 3239110656, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 3239129088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 3239147520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 3239165952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 3239184384, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 3355969536, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 3428960256, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.11.post_attention_norm.weight", "offset": 3501950976, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 3501969408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 3501987840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 3502006272, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 3506061312, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 3519037440, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 3527147520, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 3533635584, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3533654016, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 3650439168, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 3723429888, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.12.post_attention_norm.weight", "offset": 3796420608, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 3796439040, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 3796457472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 3796475904, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 3800530944, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 3813507072, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 3821617152, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 3828105216, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3828123648, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 3944908800, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 4017899520, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.13.post_attention_norm.weight", "offset": 4090890240, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 4090908672, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 4090927104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 4090945536, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 4095000576, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 4107976704, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 4116086784, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 4122574848, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4122593280, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 4239378432, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 4312369152, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.14.post_attention_norm.weight", "offset": 4385359872, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 4385378304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 4385396736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 4385415168, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 4389470208, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 4402446336, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 4410556416, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.15.attn_k.weight", "offset": 4417044480, "shape": [ 4608, 2048 ], "size": 6488064, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 4421099520, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 4434075648, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 4442185728, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 4448673792, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4448692224, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 4565477376, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 4638468096, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.15.post_attention_norm.weight", "offset": 4711458816, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 4711477248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 4711495680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 4711514112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4711532544, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 4828317696, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 4901308416, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.16.post_attention_norm.weight", "offset": 4974299136, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 4974317568, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 4974336000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4974354432, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 4978409472, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 4991385600, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 4999495680, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 5005983744, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 5006002176, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 5122787328, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 5195778048, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.17.post_attention_norm.weight", "offset": 5268768768, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 5268787200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 5268805632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 5268824064, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 5272879104, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 5285855232, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 5293965312, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 5300453376, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5300471808, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 5417256960, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 5490247680, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.18.post_attention_norm.weight", "offset": 5563238400, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 5563256832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 5563275264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 5563293696, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 5567348736, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 5580324864, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 5588434944, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 5594923008, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.19.attn_k.weight", "offset": 5667913728, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 5671968768, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 5684944896, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 5693054976, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 5699543040, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5699561472, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 5816346624, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.19.post_attention_norm.weight", "offset": 5889337344, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 5889355776, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 5889374208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 5889392640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5889411072, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 6006196224, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 6079186944, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.20.post_attention_norm.weight", "offset": 6152177664, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 6152196096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 6152214528, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 6152232960, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 6156288000, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 6169264128, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 6177374208, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 6183862272, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 6183880704, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 6300665856, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 6373656576, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.21.post_attention_norm.weight", "offset": 6446647296, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 6446665728, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 6446684160, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 6446702592, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 6450757632, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 6463733760, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 6471843840, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 6478331904, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6478350336, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 6595135488, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 6668126208, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.22.post_attention_norm.weight", "offset": 6741116928, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 6741135360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 6741153792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 6741172224, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 6745227264, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 6758203392, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 6766313472, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 6772801536, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 6845792256, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.23.attn_k.weight", "offset": 6918782976, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 6922838016, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 6935814144, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 6943924224, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 6950412288, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6950430720, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.23.post_attention_norm.weight", "offset": 7067215872, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 7067234304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 7067252736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 7067271168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 7067289600, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 7184074752, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 7257065472, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.24.post_attention_norm.weight", "offset": 7330056192, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 7330074624, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 7330093056, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 7330111488, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 7334166528, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 7347142656, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 7355252736, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 7361740800, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7361759232, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 7478544384, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 7551535104, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.25.post_attention_norm.weight", "offset": 7624525824, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 7624544256, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 7624562688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 7624581120, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 7628636160, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 7641612288, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 7649722368, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 7656210432, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7656228864, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 7773014016, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 7846004736, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.26.post_attention_norm.weight", "offset": 7918995456, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 7919013888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 7919032320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 7919050752, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 7923105792, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 7936081920, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 7944192000, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 7950680064, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7950698496, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 8067483648, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 8140474368, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.27.post_attention_norm.weight", "offset": 8213465088, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 8213483520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 8213501952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 8213520384, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 8217575424, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 8230551552, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 8238661632, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.28.attn_k.weight", "offset": 8245149696, "shape": [ 4608, 2048 ], "size": 6488064, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 8249204736, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 8262180864, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 8270290944, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 8276779008, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 8276797440, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 8393582592, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 8466573312, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.28.post_attention_norm.weight", "offset": 8539564032, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 8539582464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 8539600896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 8539619328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 8539637760, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 8656422912, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 8729413632, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.29.post_attention_norm.weight", "offset": 8802404352, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 8802422784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 8802441216, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 8802459648, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 8806514688, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 8819490816, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 8827600896, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 8834088960, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 8834107392, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 8950892544, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 9023883264, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.30.post_attention_norm.weight", "offset": 9096873984, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 9096892416, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 9096910848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 9096929280, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 9100984320, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 9113960448, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 9122070528, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 9128558592, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 9128577024, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 9245362176, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 9318352896, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.31.post_attention_norm.weight", "offset": 9391343616, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 9391362048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 9391380480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 9391398912, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 9395453952, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 9408430080, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 9416540160, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 9423028224, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.32.attn_k.weight", "offset": 9496018944, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 9500073984, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 9513050112, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 9521160192, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 9527648256, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 9527666688, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 9644451840, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.32.post_attention_norm.weight", "offset": 9717442560, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 9717460992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 9717479424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 9717497856, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 9717516288, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 9834301440, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 9907292160, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.33.post_attention_norm.weight", "offset": 9980282880, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 9980301312, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 9980319744, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 9980338176, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 9984393216, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 9997369344, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 10005479424, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 10011967488, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 10011985920, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 10128771072, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 10201761792, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.34.post_attention_norm.weight", "offset": 10274752512, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 10274770944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 10274789376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 10274807808, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 10278862848, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 10291838976, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 10299949056, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 10306437120, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 10306455552, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 10423240704, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 10496231424, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.35.post_attention_norm.weight", "offset": 10569222144, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 10569240576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 10569259008, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 10569277440, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 10573332480, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 10586308608, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 10594418688, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 10600906752, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 10673897472, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.36.attn_k.weight", "offset": 10746888192, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 10750943232, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 10763919360, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 10772029440, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 10778517504, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 10778535936, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.36.post_attention_norm.weight", "offset": 10895321088, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 10895339520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 10895357952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 10895376384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 10895394816, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 11012179968, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 11085170688, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.37.post_attention_norm.weight", "offset": 11158161408, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 11158179840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 11158198272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 11158216704, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 11162271744, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 11175247872, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 11183357952, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 11189846016, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 11189864448, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 11306649600, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 11379640320, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.38.post_attention_norm.weight", "offset": 11452631040, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 11452649472, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 11452667904, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 11452686336, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 11456741376, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 11469717504, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 11477827584, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 11484315648, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 11484334080, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 11601119232, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 11674109952, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.39.post_attention_norm.weight", "offset": 11747100672, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 11747119104, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 11747137536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 11747155968, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 11751211008, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 11764187136, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 11772297216, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.40.ffn_down.weight", "offset": 11778785280, "shape": [ 36864, 4608 ], "size": 6488064, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 11895570432, "shape": [ 4608 ], "size": 116785152, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 11895588864, "shape": [ 4608, 36864 ], "size": 18432, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 11968579584, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.40.post_attention_norm.weight", "offset": 12041570304, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 12041588736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 12041607168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 12041625600, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 12045680640, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 12058656768, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 12066766848, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.41.attn_k.weight", "offset": 12073254912, "shape": [ 4608, 2048 ], "size": 6488064, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 12077309952, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 12090286080, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 12098396160, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 12104884224, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 12104902656, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 12221687808, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 12294678528, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.41.post_attention_norm.weight", "offset": 12367669248, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 12367687680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 12367706112, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 12367724544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 12367742976, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.42.ffn_gate.weight", "offset": 12484528128, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 12557518848, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.42.post_attention_norm.weight", "offset": 12630509568, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 12630528000, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 12630546432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 12630564864, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 12634619904, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 12647596032, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 12655706112, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 12662194176, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 12662212608, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 12778997760, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 12851988480, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.43.post_attention_norm.weight", "offset": 12924979200, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 12924997632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 12925016064, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 12925034496, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 12929089536, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 12942065664, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 12950175744, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.44.attn_norm.weight", "offset": 12956663808, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 12956682240, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.44.ffn_gate.weight", "offset": 13073467392, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 13146458112, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.44.post_attention_norm.weight", "offset": 13219448832, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 13219467264, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 13219485696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 13219504128, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 13223559168, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 13236535296, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 13244645376, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 13251133440, "shape": [ 4608, 36864 ], "size": 6488064, "type": 11 }, { "name": "blk.45.attn_k.weight", "offset": 13324124160, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 13328179200, "shape": [ 4096, 4608 ], "size": 4055040, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 13341155328, "shape": [ 4608, 4096 ], "size": 12976128, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 13349265408, "shape": [ 4608, 2048 ], "size": 8110080, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 13355753472, "shape": [ 4608 ], "size": 6488064, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 13355771904, "shape": [ 36864, 4608 ], "size": 18432, "type": 13 }, { "name": "blk.45.ffn_up.weight", "offset": 13472557056, "shape": [ 4608, 36864 ], "size": 116785152, "type": 11 }, { "name": "blk.45.post_attention_norm.weight", "offset": 13545547776, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 13545566208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 13545584640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 13545603072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 13545621504, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }