gemma2

{ "metadata": { "gemma2.attention.head_count": 32, "gemma2.attention.head_count_kv": 16, "gemma2.attention.key_length": 128, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 128, "gemma2.block_count": 46, "gemma2.context_length": 8192, "gemma2.embedding_length": 4608, "gemma2.feed_forward_length": 36864, "general.architecture": "gemma2", "general.file_type": 11, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 27227128320, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 18432, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 73009152, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 145999872, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.0.post_attention_norm.weight", "offset": 218990592, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 219009024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 219027456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 219045888, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 223100928, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 231211008, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 239321088, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 243376128, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 243394560, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 316385280, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 389376000, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.1.post_attention_norm.weight", "offset": 462366720, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 462385152, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 462403584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 462422016, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 466477056, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 474587136, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 482697216, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.2.attn_k.weight", "offset": 486752256, "shape": [ 4608, 2048 ], "size": 4055040, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 490807296, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 498917376, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 507027456, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 511082496, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 511100928, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 584091648, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 657082368, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.2.post_attention_norm.weight", "offset": 730073088, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 730091520, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 730109952, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_norm.weight", "offset": 730128384, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 730146816, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 803137536, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 876128256, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.3.post_attention_norm.weight", "offset": 949118976, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 949137408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 949155840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 949174272, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 953229312, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 961339392, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 969449472, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 973504512, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 973522944, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 1046513664, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 1119504384, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.4.post_attention_norm.weight", "offset": 1192495104, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 1192513536, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 1192531968, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1192550400, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1196605440, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 1204715520, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 1212825600, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1216880640, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1216899072, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 1289889792, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 1362880512, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.5.post_attention_norm.weight", "offset": 1435871232, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 1435889664, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 1435908096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1435926528, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1439981568, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1448091648, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 1456201728, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1460256768, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.6.attn_k.weight", "offset": 1533247488, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1537302528, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1545412608, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 1553522688, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1557577728, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1557596160, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1630586880, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.6.post_attention_norm.weight", "offset": 1703577600, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 1703596032, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1703614464, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1703632896, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1703651328, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1776642048, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1849632768, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1922623488, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1922641920, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1922660352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1922678784, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1926733824, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1934843904, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 1942953984, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1947009024, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1947027456, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 2020018176, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 2093008896, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.8.post_attention_norm.weight", "offset": 2165999616, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 2166018048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 2166036480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2166054912, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 2170109952, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 2178220032, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 2186330112, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 2190385152, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2190403584, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 2263394304, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2336385024, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.9.post_attention_norm.weight", "offset": 2409375744, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 2409394176, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 2409412608, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2409431040, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 2413486080, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 2421596160, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 2429706240, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 2433761280, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 2506752000, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 2579742720, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 2583797760, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 2591907840, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 2600017920, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 2604072960, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2604091392, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.10.post_attention_norm.weight", "offset": 2677082112, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 2677100544, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 2677118976, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 2677137408, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2677155840, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 2750146560, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 2823137280, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.11.post_attention_norm.weight", "offset": 2896128000, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 2896146432, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 2896164864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 2896183296, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 2900238336, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 2908348416, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 2916458496, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2920513536, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2920531968, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 2993522688, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 3066513408, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.12.post_attention_norm.weight", "offset": 3139504128, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 3139522560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 3139540992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 3139559424, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 3143614464, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 3151724544, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 3159834624, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 3163889664, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3163908096, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 3236898816, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 3309889536, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.13.post_attention_norm.weight", "offset": 3382880256, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 3382898688, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 3382917120, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3382935552, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 3386990592, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 3395100672, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 3403210752, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 3407265792, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3407284224, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 3480274944, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 3553265664, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.14.post_attention_norm.weight", "offset": 3626256384, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 3626274816, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 3626293248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3626311680, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 3630366720, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 3638476800, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 3646586880, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.15.attn_k.weight", "offset": 3650641920, "shape": [ 4608, 2048 ], "size": 4055040, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 3654696960, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 3662807040, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 3670917120, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 3674972160, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3674990592, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 3747981312, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 3820972032, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.15.post_attention_norm.weight", "offset": 3893962752, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 3893981184, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 3893999616, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 3894018048, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3894036480, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 3967027200, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 4040017920, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.16.post_attention_norm.weight", "offset": 4113008640, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 4113027072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 4113045504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4113063936, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 4117118976, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 4125229056, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 4133339136, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 4137394176, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4137412608, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 4210403328, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 4283394048, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.17.post_attention_norm.weight", "offset": 4356384768, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 4356403200, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 4356421632, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 4356440064, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 4360495104, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 4368605184, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 4376715264, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 4380770304, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4380788736, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 4453779456, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 4526770176, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.18.post_attention_norm.weight", "offset": 4599760896, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 4599779328, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 4599797760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 4599816192, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 4603871232, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 4611981312, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 4620091392, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 4624146432, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.19.attn_k.weight", "offset": 4697137152, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 4701192192, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 4709302272, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 4717412352, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 4721467392, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4721485824, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 4794476544, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.19.post_attention_norm.weight", "offset": 4867467264, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 4867485696, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 4867504128, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_norm.weight", "offset": 4867522560, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4867540992, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 4940531712, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 5013522432, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.20.post_attention_norm.weight", "offset": 5086513152, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 5086531584, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 5086550016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 5086568448, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 5090623488, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 5098733568, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 5106843648, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 5110898688, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5110917120, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 5183907840, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 5256898560, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.21.post_attention_norm.weight", "offset": 5329889280, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 5329907712, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 5329926144, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 5329944576, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 5333999616, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 5342109696, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 5350219776, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 5354274816, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5354293248, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 5427283968, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 5500274688, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.22.post_attention_norm.weight", "offset": 5573265408, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 5573283840, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 5573302272, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 5573320704, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 5577375744, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 5585485824, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 5593595904, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 5597650944, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 5670641664, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.23.attn_k.weight", "offset": 5743632384, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 5747687424, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 5755797504, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 5763907584, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 5767962624, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5767981056, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.23.post_attention_norm.weight", "offset": 5840971776, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 5840990208, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 5841008640, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_norm.weight", "offset": 5841027072, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5841045504, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 5914036224, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 5987026944, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.24.post_attention_norm.weight", "offset": 6060017664, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 6060036096, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 6060054528, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 6060072960, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 6064128000, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 6072238080, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 6080348160, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 6084403200, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6084421632, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 6157412352, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 6230403072, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.25.post_attention_norm.weight", "offset": 6303393792, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 6303412224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 6303430656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 6303449088, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 6307504128, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 6315614208, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 6323724288, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 6327779328, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6327797760, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 6400788480, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 6473779200, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.26.post_attention_norm.weight", "offset": 6546769920, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 6546788352, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 6546806784, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6546825216, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 6550880256, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 6558990336, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 6567100416, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 6571155456, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6571173888, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 6644164608, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 6717155328, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.27.post_attention_norm.weight", "offset": 6790146048, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 6790164480, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 6790182912, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6790201344, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 6794256384, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 6802366464, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 6810476544, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.28.attn_k.weight", "offset": 6814531584, "shape": [ 4608, 2048 ], "size": 4055040, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 6818586624, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 6826696704, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 6834806784, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 6838861824, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6838880256, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 6911870976, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 6984861696, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.28.post_attention_norm.weight", "offset": 7057852416, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 7057870848, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 7057889280, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 7057907712, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7057926144, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 7130916864, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 7203907584, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.29.post_attention_norm.weight", "offset": 7276898304, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 7276916736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 7276935168, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 7276953600, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 7281008640, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 7289118720, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 7297228800, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 7301283840, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7301302272, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 7374292992, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 7447283712, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.30.post_attention_norm.weight", "offset": 7520274432, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 7520292864, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 7520311296, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 7520329728, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 7524384768, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 7532494848, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 7540604928, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 7544659968, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7544678400, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 7617669120, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 7690659840, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.31.post_attention_norm.weight", "offset": 7763650560, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 7763668992, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 7763687424, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 7763705856, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 7767760896, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 7775870976, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 7783981056, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 7788036096, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.32.attn_k.weight", "offset": 7861026816, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 7865081856, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 7873191936, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 7881302016, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 7885357056, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7885375488, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 7958366208, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.32.post_attention_norm.weight", "offset": 8031356928, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 8031375360, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 8031393792, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 8031412224, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8031430656, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 8104421376, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 8177412096, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.33.post_attention_norm.weight", "offset": 8250402816, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 8250421248, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 8250439680, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 8250458112, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 8254513152, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 8262623232, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 8270733312, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 8274788352, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8274806784, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 8347797504, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 8420788224, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.34.post_attention_norm.weight", "offset": 8493778944, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 8493797376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_norm.weight", "offset": 8493815808, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 8493834240, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 8497889280, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 8505999360, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 8514109440, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 8518164480, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8518182912, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 8591173632, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 8664164352, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.35.post_attention_norm.weight", "offset": 8737155072, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 8737173504, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 8737191936, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 8737210368, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 8741265408, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 8749375488, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 8757485568, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 8761540608, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 8834531328, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.36.attn_k.weight", "offset": 8907522048, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 8911577088, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 8919687168, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 8927797248, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 8931852288, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8931870720, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.36.post_attention_norm.weight", "offset": 9004861440, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 9004879872, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.36.ffn_norm.weight", "offset": 9004898304, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_norm.weight", "offset": 9004916736, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9004935168, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 9077925888, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 9150916608, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.37.post_attention_norm.weight", "offset": 9223907328, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 9223925760, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 9223944192, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 9223962624, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 9228017664, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 9236127744, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 9244237824, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 9248292864, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9248311296, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 9321302016, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 9394292736, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.38.post_attention_norm.weight", "offset": 9467283456, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 9467301888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 9467320320, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 9467338752, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 9471393792, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 9479503872, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 9487613952, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 9491668992, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9491687424, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 9564678144, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 9637668864, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.39.post_attention_norm.weight", "offset": 9710659584, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 9710678016, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 9710696448, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 9710714880, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 9714769920, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 9722880000, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 9730990080, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.40.ffn_down.weight", "offset": 9735045120, "shape": [ 36864, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 9808035840, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.40.ffn_gate.weight", "offset": 9808054272, "shape": [ 4608, 36864 ], "size": 18432, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 9881044992, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.40.post_attention_norm.weight", "offset": 9954035712, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 9954054144, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 9954072576, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 9954091008, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 9958146048, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 9966256128, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 9974366208, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.41.attn_k.weight", "offset": 9978421248, "shape": [ 4608, 2048 ], "size": 4055040, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 9982476288, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 9990586368, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 9998696448, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 10002751488, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 10002769920, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.41.ffn_gate.weight", "offset": 10075760640, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 10148751360, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.41.post_attention_norm.weight", "offset": 10221742080, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 10221760512, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 10221778944, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 10221797376, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 10221815808, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.42.ffn_gate.weight", "offset": 10294806528, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 10367797248, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.42.post_attention_norm.weight", "offset": 10440787968, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.42.post_ffw_norm.weight", "offset": 10440806400, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.ffn_norm.weight", "offset": 10440824832, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 10440843264, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 10444898304, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 10453008384, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 10461118464, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 10465173504, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 10465191936, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.43.ffn_gate.weight", "offset": 10538182656, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 10611173376, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.43.post_attention_norm.weight", "offset": 10684164096, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.43.post_ffw_norm.weight", "offset": 10684182528, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.ffn_norm.weight", "offset": 10684200960, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 10684219392, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 10688274432, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 10696384512, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 10704494592, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.44.attn_norm.weight", "offset": 10708549632, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 10708568064, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.44.ffn_gate.weight", "offset": 10781558784, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 10854549504, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.44.post_attention_norm.weight", "offset": 10927540224, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.44.post_ffw_norm.weight", "offset": 10927558656, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.ffn_norm.weight", "offset": 10927577088, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 10927595520, "shape": [ 4608, 2048 ], "size": 18432, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 10931650560, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 10939760640, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 10947870720, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.45.ffn_gate.weight", "offset": 10951925760, "shape": [ 4608, 36864 ], "size": 4055040, "type": 11 }, { "name": "blk.45.attn_k.weight", "offset": 11024916480, "shape": [ 4608, 2048 ], "size": 72990720, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 11028971520, "shape": [ 4096, 4608 ], "size": 4055040, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 11037081600, "shape": [ 4608, 4096 ], "size": 8110080, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 11045191680, "shape": [ 4608, 2048 ], "size": 8110080, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 11049246720, "shape": [ 4608 ], "size": 4055040, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 11049265152, "shape": [ 36864, 4608 ], "size": 18432, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 11122255872, "shape": [ 4608, 36864 ], "size": 72990720, "type": 11 }, { "name": "blk.45.post_attention_norm.weight", "offset": 11195246592, "shape": [ 4608 ], "size": 72990720, "type": 0 }, { "name": "blk.45.post_ffw_norm.weight", "offset": 11195265024, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "blk.45.ffn_norm.weight", "offset": 11195283456, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "output_norm.weight", "offset": 11195301888, "shape": [ 4608 ], "size": 18432, "type": 0 }, { "name": "token_embd.weight", "offset": 11195320320, "shape": [ 4608, 256000 ], "size": 18432, "type": 14 } ], "version": 3 }