llama2

{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "LLaMA v2", "general.quantization_version": 2, "llama.attention.head_count": 64, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 80, "llama.context_length": 4096, "llama.embedding_length": 8192, "llama.feed_forward_length": 28672, "llama.rope.dimension_count": 128, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.merges": "... (61249 values)", "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32000 values)", "tokenizer.ggml.token_type": "... (32000 values)", "tokenizer.ggml.tokens": "... (32000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 68976648192, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 147456000, "shape": [ 8192 ], "size": 147456000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 147488768, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 279609344, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 411729920, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 543850496, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 543883264, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 548601856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 586350592, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 624099328, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 628817920, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 628850688, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 760971264, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 893091840, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 1025212416, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1025245184, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 1029963776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 1067712512, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 1105461248, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 1110179840, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1110212608, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 1242333184, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 1374453760, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 1506574336, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1506607104, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 1511325696, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 1549074432, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 1586823168, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 1591541760, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1591574528, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 1723695104, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 1855815680, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 1987936256, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1987969024, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 1992687616, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 2030436352, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 2068185088, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 2072903680, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 2072936448, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 2205057024, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 2337177600, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 2469298176, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 2469330944, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 2474049536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 2511798272, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 2549547008, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 2554265600, "shape": [ 8192, 28672 ], "size": 4718592, "type": 2 }, { "name": "blk.5.attn_k.weight", "offset": 2686386176, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 2691104768, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 2728853504, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 2766602240, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 2771320832, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2771353600, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 2903474176, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 3035594752, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 3167715328, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 3167748096, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 3172466688, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 3210215424, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 3247964160, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.11.attn_k.weight", "offset": 3252682752, "shape": [ 8192, 1024 ], "size": 4718592, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 3257401344, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 3295150080, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 3332898816, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 3337617408, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3337650176, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 3469770752, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 3601891328, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.6.attn_norm.weight", "offset": 3601924096, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3601956864, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 3734077440, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 3866198016, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 3998318592, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 3998351360, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 4003069952, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 4040818688, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 4078567424, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 4083286016, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4083318784, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 4215439360, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 4347559936, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 4479680512, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 4479713280, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 4484431872, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 4522180608, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 4559929344, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 4564647936, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4564680704, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 4696801280, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 4828921856, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 4961042432, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 4961075200, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 4965793792, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 5003542528, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 5041291264, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 5046009856, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 5046042624, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 5178163200, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 5310283776, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 5442404352, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 5442437120, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 5447155712, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 5484904448, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 5522653184, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 5527371776, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5527404544, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 5659525120, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 5791645696, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 5923766272, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.12.attn_norm.weight", "offset": 5923799040, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 5923831808, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 6055952384, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 6188072960, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 6320193536, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 6320226304, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 6324944896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 6362693632, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 6400442368, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 6405160960, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6405193728, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 6537314304, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 6669434880, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 6801555456, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 6801588224, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 6806306816, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 6844055552, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 6881804288, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 6886522880, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 6886555648, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 7018676224, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 7150796800, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 7282917376, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 7282950144, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 7287668736, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 7325417472, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 7363166208, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 7367884800, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 7367917568, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 7500038144, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 7632158720, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 7764279296, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 7764312064, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 7769030656, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 7806779392, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 7844528128, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 7849246720, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 7849279488, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 7981400064, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 8113520640, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 8245641216, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 8245673984, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 8250392576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 8288141312, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 8325890048, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 8330608640, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 8330641408, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 8462761984, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 8594882560, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 8727003136, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 8727035904, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 8731754496, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 8769503232, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 8807251968, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 8811970560, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 8812003328, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 8944123904, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 9076244480, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 9208365056, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 9208397824, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 9213116416, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 9250865152, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 9288613888, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 9293332480, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 9293365248, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 9425485824, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 9557606400, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 9689726976, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 9689759744, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 9694478336, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 9732227072, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 9769975808, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 9774694400, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 9774727168, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 9906847744, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 10038968320, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 10171088896, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 10171121664, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 10175840256, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 10213588992, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 10251337728, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 10256056320, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 10256089088, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 10388209664, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 10520330240, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 10652450816, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 10652483584, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 10657202176, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 10694950912, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 10732699648, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.22.ffn_down.weight", "offset": 10737418240, "shape": [ 28672, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 10869538816, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.22.attn_k.weight", "offset": 11001659392, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 11006377984, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 11044126720, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 11081875456, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 11086594048, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.22.ffn_up.weight", "offset": 11086626816, "shape": [ 8192, 28672 ], "size": 32768, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 11218747392, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 11218780160, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 11218812928, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 11350933504, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 11483054080, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 11615174656, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 11615207424, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 11619926016, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 11657674752, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 11695423488, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 11700142080, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 11700174848, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 11832295424, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 11964416000, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 12096536576, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 12096569344, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 12101287936, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 12139036672, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 12176785408, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 12181504000, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 12181536768, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 12313657344, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 12445777920, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 12577898496, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 12577931264, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 12582649856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 12620398592, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 12658147328, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 12662865920, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 12662898688, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 12795019264, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 12927139840, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 13059260416, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 13059293184, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 13064011776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 13101760512, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 13139509248, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 13144227840, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 13144260608, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 13276381184, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 13408501760, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 13540622336, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 13540655104, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 13545373696, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 13583122432, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 13620871168, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 13625589760, "shape": [ 8192, 28672 ], "size": 4718592, "type": 2 }, { "name": "blk.28.attn_k.weight", "offset": 13757710336, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 13762428928, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 13800177664, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 13837926400, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 13842644992, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 13842677760, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 13974798336, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 14106918912, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 14106951680, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 14106984448, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 14239105024, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 14371225600, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 14503346176, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 14503378944, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 14508097536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 14545846272, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 14583595008, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 14588313600, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 14588346368, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 14720466944, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 14852587520, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 14984708096, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 14984740864, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 14989459456, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 15027208192, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 15064956928, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 15069675520, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 15069708288, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 15201828864, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 15333949440, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 15466070016, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 15466102784, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 15470821376, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 15508570112, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 15546318848, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 15551037440, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 15551070208, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.32.ffn_gate.weight", "offset": 15683190784, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 15815311360, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 15947431936, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 15947464704, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.32.attn_output.weight", "offset": 15952183296, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.32.attn_q.weight", "offset": 15989932032, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.32.attn_v.weight", "offset": 16027680768, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 16032399360, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 16032432128, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.33.ffn_gate.weight", "offset": 16164552704, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 16296673280, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 16428793856, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 16428826624, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.33.attn_output.weight", "offset": 16433545216, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.33.attn_q.weight", "offset": 16471293952, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.33.attn_v.weight", "offset": 16509042688, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.34.attn_k.weight", "offset": 16513761280, "shape": [ 8192, 1024 ], "size": 4718592, "type": 2 }, { "name": "blk.34.attn_output.weight", "offset": 16518479872, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.34.attn_q.weight", "offset": 16556228608, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.34.attn_v.weight", "offset": 16593977344, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 16598695936, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 16598728704, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.34.ffn_gate.weight", "offset": 16730849280, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 16862969856, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 16995090432, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.35.attn_norm.weight", "offset": 16995123200, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 16995155968, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.35.ffn_gate.weight", "offset": 17127276544, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 17259397120, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 17391517696, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 17391550464, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 17396269056, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.35.attn_q.weight", "offset": 17434017792, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.35.attn_v.weight", "offset": 17471766528, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.36.attn_norm.weight", "offset": 17476485120, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 17476517888, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.36.ffn_gate.weight", "offset": 17608638464, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 17740759040, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 17872879616, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 17872912384, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.36.attn_output.weight", "offset": 17877630976, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.36.attn_q.weight", "offset": 17915379712, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.36.attn_v.weight", "offset": 17953128448, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 17957847040, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 17957879808, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.37.ffn_gate.weight", "offset": 18090000384, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 18222120960, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 18354241536, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 18354274304, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.37.attn_output.weight", "offset": 18358992896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.37.attn_q.weight", "offset": 18396741632, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.37.attn_v.weight", "offset": 18434490368, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 18439208960, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 18439241728, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.38.ffn_gate.weight", "offset": 18571362304, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 18703482880, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 18835603456, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 18835636224, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.38.attn_output.weight", "offset": 18840354816, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.38.attn_q.weight", "offset": 18878103552, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.38.attn_v.weight", "offset": 18915852288, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 18920570880, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 18920603648, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.39.ffn_gate.weight", "offset": 19052724224, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 19184844800, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 19316965376, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 19316998144, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.39.attn_output.weight", "offset": 19321716736, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.39.attn_q.weight", "offset": 19359465472, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.39.attn_v.weight", "offset": 19397214208, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 19401932800, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 19401965568, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.40.ffn_gate.weight", "offset": 19534086144, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.40.ffn_up.weight", "offset": 19666206720, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.40.ffn_norm.weight", "offset": 19798327296, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 19798360064, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.40.attn_output.weight", "offset": 19803078656, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.40.attn_q.weight", "offset": 19840827392, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.40.attn_v.weight", "offset": 19878576128, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 19883294720, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 19883327488, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.41.ffn_gate.weight", "offset": 20015448064, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.41.ffn_up.weight", "offset": 20147568640, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.41.ffn_norm.weight", "offset": 20279689216, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 20279721984, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.41.attn_output.weight", "offset": 20284440576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.41.attn_q.weight", "offset": 20322189312, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.41.attn_v.weight", "offset": 20359938048, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.42.attn_norm.weight", "offset": 20364656640, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 20364689408, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.42.ffn_gate.weight", "offset": 20496809984, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.42.ffn_up.weight", "offset": 20628930560, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.42.ffn_norm.weight", "offset": 20761051136, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 20761083904, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.42.attn_output.weight", "offset": 20765802496, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.42.attn_q.weight", "offset": 20803551232, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.42.attn_v.weight", "offset": 20841299968, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.43.attn_norm.weight", "offset": 20846018560, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 20846051328, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.43.ffn_gate.weight", "offset": 20978171904, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.43.ffn_up.weight", "offset": 21110292480, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.43.ffn_norm.weight", "offset": 21242413056, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 21242445824, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.43.attn_output.weight", "offset": 21247164416, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.43.attn_q.weight", "offset": 21284913152, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.43.attn_v.weight", "offset": 21322661888, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.44.attn_norm.weight", "offset": 21327380480, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 21327413248, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.44.ffn_gate.weight", "offset": 21459533824, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.44.ffn_up.weight", "offset": 21591654400, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.44.ffn_norm.weight", "offset": 21723774976, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 21723807744, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.44.attn_output.weight", "offset": 21728526336, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.44.attn_q.weight", "offset": 21766275072, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.44.attn_v.weight", "offset": 21804023808, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.45.ffn_down.weight", "offset": 21808742400, "shape": [ 28672, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.45.ffn_gate.weight", "offset": 21940862976, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.45.attn_k.weight", "offset": 22072983552, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.45.attn_output.weight", "offset": 22077702144, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.45.attn_q.weight", "offset": 22115450880, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.45.attn_v.weight", "offset": 22153199616, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.45.attn_norm.weight", "offset": 22157918208, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.45.ffn_up.weight", "offset": 22157950976, "shape": [ 8192, 28672 ], "size": 32768, "type": 2 }, { "name": "blk.45.ffn_norm.weight", "offset": 22290071552, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.46.attn_norm.weight", "offset": 22290104320, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 22290137088, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.46.ffn_gate.weight", "offset": 22422257664, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.46.ffn_up.weight", "offset": 22554378240, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.46.ffn_norm.weight", "offset": 22686498816, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 22686531584, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.46.attn_output.weight", "offset": 22691250176, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.46.attn_q.weight", "offset": 22728998912, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.46.attn_v.weight", "offset": 22766747648, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.47.attn_norm.weight", "offset": 22771466240, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 22771499008, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.47.ffn_gate.weight", "offset": 22903619584, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.47.ffn_up.weight", "offset": 23035740160, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.47.ffn_norm.weight", "offset": 23167860736, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 23167893504, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.47.attn_output.weight", "offset": 23172612096, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.47.attn_q.weight", "offset": 23210360832, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.47.attn_v.weight", "offset": 23248109568, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.48.attn_norm.weight", "offset": 23252828160, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.48.ffn_down.weight", "offset": 23252860928, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.48.ffn_gate.weight", "offset": 23384981504, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.48.ffn_up.weight", "offset": 23517102080, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.48.ffn_norm.weight", "offset": 23649222656, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.48.attn_k.weight", "offset": 23649255424, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.48.attn_output.weight", "offset": 23653974016, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.48.attn_q.weight", "offset": 23691722752, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.48.attn_v.weight", "offset": 23729471488, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.49.attn_norm.weight", "offset": 23734190080, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.49.ffn_down.weight", "offset": 23734222848, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.49.ffn_gate.weight", "offset": 23866343424, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.49.ffn_up.weight", "offset": 23998464000, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.49.ffn_norm.weight", "offset": 24130584576, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.49.attn_k.weight", "offset": 24130617344, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.49.attn_output.weight", "offset": 24135335936, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.49.attn_q.weight", "offset": 24173084672, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.49.attn_v.weight", "offset": 24210833408, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.50.attn_norm.weight", "offset": 24215552000, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.50.ffn_down.weight", "offset": 24215584768, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.50.ffn_gate.weight", "offset": 24347705344, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.50.ffn_up.weight", "offset": 24479825920, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.50.ffn_norm.weight", "offset": 24611946496, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.50.attn_k.weight", "offset": 24611979264, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.50.attn_output.weight", "offset": 24616697856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.50.attn_q.weight", "offset": 24654446592, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.50.attn_v.weight", "offset": 24692195328, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.51.ffn_gate.weight", "offset": 24696913920, "shape": [ 8192, 28672 ], "size": 4718592, "type": 2 }, { "name": "blk.51.attn_k.weight", "offset": 24829034496, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.51.attn_output.weight", "offset": 24833753088, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.51.attn_q.weight", "offset": 24871501824, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.51.attn_v.weight", "offset": 24909250560, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.51.attn_norm.weight", "offset": 24913969152, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.51.ffn_down.weight", "offset": 24914001920, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.51.ffn_up.weight", "offset": 25046122496, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.51.ffn_norm.weight", "offset": 25178243072, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.52.attn_norm.weight", "offset": 25178275840, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.52.ffn_down.weight", "offset": 25178308608, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.52.ffn_gate.weight", "offset": 25310429184, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.52.ffn_up.weight", "offset": 25442549760, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.52.ffn_norm.weight", "offset": 25574670336, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.52.attn_k.weight", "offset": 25574703104, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.52.attn_output.weight", "offset": 25579421696, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.52.attn_q.weight", "offset": 25617170432, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.52.attn_v.weight", "offset": 25654919168, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.53.attn_norm.weight", "offset": 25659637760, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.53.ffn_down.weight", "offset": 25659670528, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.53.ffn_gate.weight", "offset": 25791791104, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.53.ffn_up.weight", "offset": 25923911680, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.53.ffn_norm.weight", "offset": 26056032256, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.53.attn_k.weight", "offset": 26056065024, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.53.attn_output.weight", "offset": 26060783616, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.53.attn_q.weight", "offset": 26098532352, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.53.attn_v.weight", "offset": 26136281088, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.54.attn_norm.weight", "offset": 26140999680, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.54.ffn_down.weight", "offset": 26141032448, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.54.ffn_gate.weight", "offset": 26273153024, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.54.ffn_up.weight", "offset": 26405273600, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.54.ffn_norm.weight", "offset": 26537394176, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.54.attn_k.weight", "offset": 26537426944, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.54.attn_output.weight", "offset": 26542145536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.54.attn_q.weight", "offset": 26579894272, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.54.attn_v.weight", "offset": 26617643008, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.55.attn_norm.weight", "offset": 26622361600, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.55.ffn_down.weight", "offset": 26622394368, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.55.ffn_gate.weight", "offset": 26754514944, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.55.ffn_up.weight", "offset": 26886635520, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.55.ffn_norm.weight", "offset": 27018756096, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.55.attn_k.weight", "offset": 27018788864, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.55.attn_output.weight", "offset": 27023507456, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.55.attn_q.weight", "offset": 27061256192, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.55.attn_v.weight", "offset": 27099004928, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.56.attn_norm.weight", "offset": 27103723520, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.56.ffn_down.weight", "offset": 27103756288, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.56.ffn_gate.weight", "offset": 27235876864, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.56.ffn_up.weight", "offset": 27367997440, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.56.ffn_norm.weight", "offset": 27500118016, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.56.attn_k.weight", "offset": 27500150784, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.56.attn_output.weight", "offset": 27504869376, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.56.attn_q.weight", "offset": 27542618112, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.56.attn_v.weight", "offset": 27580366848, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.57.attn_k.weight", "offset": 27585085440, "shape": [ 8192, 1024 ], "size": 4718592, "type": 2 }, { "name": "blk.57.attn_output.weight", "offset": 27589804032, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.57.attn_q.weight", "offset": 27627552768, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.57.attn_v.weight", "offset": 27665301504, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.57.attn_norm.weight", "offset": 27670020096, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.57.ffn_down.weight", "offset": 27670052864, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.57.ffn_gate.weight", "offset": 27802173440, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.57.ffn_up.weight", "offset": 27934294016, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.57.ffn_norm.weight", "offset": 28066414592, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.58.attn_norm.weight", "offset": 28066447360, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.58.ffn_down.weight", "offset": 28066480128, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.58.ffn_gate.weight", "offset": 28198600704, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.58.ffn_up.weight", "offset": 28330721280, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.58.ffn_norm.weight", "offset": 28462841856, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.58.attn_k.weight", "offset": 28462874624, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.58.attn_output.weight", "offset": 28467593216, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.58.attn_q.weight", "offset": 28505341952, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.58.attn_v.weight", "offset": 28543090688, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.59.attn_norm.weight", "offset": 28547809280, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.59.ffn_down.weight", "offset": 28547842048, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.59.ffn_gate.weight", "offset": 28679962624, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.59.ffn_up.weight", "offset": 28812083200, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.59.ffn_norm.weight", "offset": 28944203776, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.59.attn_k.weight", "offset": 28944236544, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.59.attn_output.weight", "offset": 28948955136, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.59.attn_q.weight", "offset": 28986703872, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.59.attn_v.weight", "offset": 29024452608, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.60.attn_norm.weight", "offset": 29029171200, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.60.ffn_down.weight", "offset": 29029203968, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.60.ffn_gate.weight", "offset": 29161324544, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.60.ffn_up.weight", "offset": 29293445120, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.60.ffn_norm.weight", "offset": 29425565696, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.60.attn_k.weight", "offset": 29425598464, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.60.attn_output.weight", "offset": 29430317056, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.60.attn_q.weight", "offset": 29468065792, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.60.attn_v.weight", "offset": 29505814528, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.61.attn_norm.weight", "offset": 29510533120, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.61.ffn_down.weight", "offset": 29510565888, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.61.ffn_gate.weight", "offset": 29642686464, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.61.ffn_up.weight", "offset": 29774807040, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.61.ffn_norm.weight", "offset": 29906927616, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.61.attn_k.weight", "offset": 29906960384, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.61.attn_output.weight", "offset": 29911678976, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.61.attn_q.weight", "offset": 29949427712, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.61.attn_v.weight", "offset": 29987176448, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.62.attn_norm.weight", "offset": 29991895040, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.62.ffn_down.weight", "offset": 29991927808, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.62.ffn_gate.weight", "offset": 30124048384, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.62.ffn_up.weight", "offset": 30256168960, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.62.ffn_norm.weight", "offset": 30388289536, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.62.attn_k.weight", "offset": 30388322304, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.62.attn_output.weight", "offset": 30393040896, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.62.attn_q.weight", "offset": 30430789632, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.62.attn_v.weight", "offset": 30468538368, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.63.attn_norm.weight", "offset": 30473256960, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.63.ffn_down.weight", "offset": 30473289728, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.63.ffn_gate.weight", "offset": 30605410304, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.63.ffn_up.weight", "offset": 30737530880, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.63.ffn_norm.weight", "offset": 30869651456, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.63.attn_k.weight", "offset": 30869684224, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.63.attn_output.weight", "offset": 30874402816, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.63.attn_q.weight", "offset": 30912151552, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.63.attn_v.weight", "offset": 30949900288, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.64.attn_norm.weight", "offset": 30954618880, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.64.ffn_down.weight", "offset": 30954651648, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.64.ffn_gate.weight", "offset": 31086772224, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.64.ffn_up.weight", "offset": 31218892800, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.64.ffn_norm.weight", "offset": 31351013376, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.64.attn_k.weight", "offset": 31351046144, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.64.attn_output.weight", "offset": 31355764736, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.64.attn_q.weight", "offset": 31393513472, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.64.attn_v.weight", "offset": 31431262208, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.65.attn_norm.weight", "offset": 31435980800, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.65.ffn_down.weight", "offset": 31436013568, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.65.ffn_gate.weight", "offset": 31568134144, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.65.ffn_up.weight", "offset": 31700254720, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.65.ffn_norm.weight", "offset": 31832375296, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.65.attn_k.weight", "offset": 31832408064, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.65.attn_output.weight", "offset": 31837126656, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.65.attn_q.weight", "offset": 31874875392, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.65.attn_v.weight", "offset": 31912624128, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.66.attn_norm.weight", "offset": 31917342720, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.66.ffn_down.weight", "offset": 31917375488, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.66.ffn_gate.weight", "offset": 32049496064, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.66.ffn_up.weight", "offset": 32181616640, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.66.ffn_norm.weight", "offset": 32313737216, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.66.attn_k.weight", "offset": 32313769984, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.66.attn_output.weight", "offset": 32318488576, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.66.attn_q.weight", "offset": 32356237312, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.66.attn_v.weight", "offset": 32393986048, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.67.attn_norm.weight", "offset": 32398704640, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.67.ffn_down.weight", "offset": 32398737408, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.67.ffn_gate.weight", "offset": 32530857984, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.67.ffn_up.weight", "offset": 32662978560, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.67.ffn_norm.weight", "offset": 32795099136, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.67.attn_k.weight", "offset": 32795131904, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.67.attn_output.weight", "offset": 32799850496, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.67.attn_q.weight", "offset": 32837599232, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.67.attn_v.weight", "offset": 32875347968, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.68.ffn_down.weight", "offset": 32880066560, "shape": [ 28672, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.68.ffn_gate.weight", "offset": 33012187136, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.68.attn_k.weight", "offset": 33144307712, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.68.attn_output.weight", "offset": 33149026304, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.68.attn_q.weight", "offset": 33186775040, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.68.attn_v.weight", "offset": 33224523776, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.68.attn_norm.weight", "offset": 33229242368, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.68.ffn_up.weight", "offset": 33229275136, "shape": [ 8192, 28672 ], "size": 32768, "type": 2 }, { "name": "blk.68.ffn_norm.weight", "offset": 33361395712, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.69.attn_norm.weight", "offset": 33361428480, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.69.ffn_down.weight", "offset": 33361461248, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.69.ffn_gate.weight", "offset": 33493581824, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.69.ffn_up.weight", "offset": 33625702400, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.69.ffn_norm.weight", "offset": 33757822976, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.69.attn_k.weight", "offset": 33757855744, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.69.attn_output.weight", "offset": 33762574336, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.69.attn_q.weight", "offset": 33800323072, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.69.attn_v.weight", "offset": 33838071808, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.70.attn_norm.weight", "offset": 33842790400, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.70.ffn_down.weight", "offset": 33842823168, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.70.ffn_gate.weight", "offset": 33974943744, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.70.ffn_up.weight", "offset": 34107064320, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.70.ffn_norm.weight", "offset": 34239184896, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.70.attn_k.weight", "offset": 34239217664, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.70.attn_output.weight", "offset": 34243936256, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.70.attn_q.weight", "offset": 34281684992, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.70.attn_v.weight", "offset": 34319433728, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.71.attn_norm.weight", "offset": 34324152320, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.71.ffn_down.weight", "offset": 34324185088, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.71.ffn_gate.weight", "offset": 34456305664, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.71.ffn_up.weight", "offset": 34588426240, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.71.ffn_norm.weight", "offset": 34720546816, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.71.attn_k.weight", "offset": 34720579584, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.71.attn_output.weight", "offset": 34725298176, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.71.attn_q.weight", "offset": 34763046912, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.71.attn_v.weight", "offset": 34800795648, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.72.attn_norm.weight", "offset": 34805514240, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.72.ffn_down.weight", "offset": 34805547008, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.72.ffn_gate.weight", "offset": 34937667584, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.72.ffn_up.weight", "offset": 35069788160, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.72.ffn_norm.weight", "offset": 35201908736, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.72.attn_k.weight", "offset": 35201941504, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.72.attn_output.weight", "offset": 35206660096, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.72.attn_q.weight", "offset": 35244408832, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.72.attn_v.weight", "offset": 35282157568, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.73.attn_norm.weight", "offset": 35286876160, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.73.ffn_down.weight", "offset": 35286908928, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.73.ffn_gate.weight", "offset": 35419029504, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.73.ffn_up.weight", "offset": 35551150080, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.73.ffn_norm.weight", "offset": 35683270656, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.73.attn_k.weight", "offset": 35683303424, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.73.attn_output.weight", "offset": 35688022016, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.73.attn_q.weight", "offset": 35725770752, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.73.attn_v.weight", "offset": 35763519488, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.74.ffn_gate.weight", "offset": 35768238080, "shape": [ 8192, 28672 ], "size": 4718592, "type": 2 }, { "name": "blk.74.attn_k.weight", "offset": 35900358656, "shape": [ 8192, 1024 ], "size": 132120576, "type": 2 }, { "name": "blk.74.attn_output.weight", "offset": 35905077248, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.74.attn_q.weight", "offset": 35942825984, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.74.attn_v.weight", "offset": 35980574720, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.74.attn_norm.weight", "offset": 35985293312, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.74.ffn_down.weight", "offset": 35985326080, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.74.ffn_up.weight", "offset": 36117446656, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.74.ffn_norm.weight", "offset": 36249567232, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.75.attn_norm.weight", "offset": 36249600000, "shape": [ 8192 ], "size": 32768, "type": 0 }, { "name": "blk.75.ffn_down.weight", "offset": 36249632768, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.75.ffn_gate.weight", "offset": 36381753344, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.75.ffn_up.weight", "offset": 36513873920, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.75.ffn_norm.weight", "offset": 36645994496, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.75.attn_k.weight", "offset": 36646027264, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.75.attn_output.weight", "offset": 36650745856, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.75.attn_q.weight", "offset": 36688494592, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.75.attn_v.weight", "offset": 36726243328, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.76.attn_norm.weight", "offset": 36730961920, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.76.ffn_down.weight", "offset": 36730994688, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.76.ffn_gate.weight", "offset": 36863115264, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.76.ffn_up.weight", "offset": 36995235840, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.76.ffn_norm.weight", "offset": 37127356416, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.76.attn_k.weight", "offset": 37127389184, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.76.attn_output.weight", "offset": 37132107776, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.76.attn_q.weight", "offset": 37169856512, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.76.attn_v.weight", "offset": 37207605248, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.77.attn_norm.weight", "offset": 37212323840, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.77.ffn_down.weight", "offset": 37212356608, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.77.ffn_gate.weight", "offset": 37344477184, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.77.ffn_up.weight", "offset": 37476597760, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.77.ffn_norm.weight", "offset": 37608718336, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.77.attn_k.weight", "offset": 37608751104, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.77.attn_output.weight", "offset": 37613469696, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.77.attn_q.weight", "offset": 37651218432, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.77.attn_v.weight", "offset": 37688967168, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.78.attn_norm.weight", "offset": 37693685760, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.78.ffn_down.weight", "offset": 37693718528, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.78.ffn_gate.weight", "offset": 37825839104, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.78.ffn_up.weight", "offset": 37957959680, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.78.ffn_norm.weight", "offset": 38090080256, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.78.attn_k.weight", "offset": 38090113024, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.78.attn_output.weight", "offset": 38094831616, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.78.attn_q.weight", "offset": 38132580352, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.78.attn_v.weight", "offset": 38170329088, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "blk.79.attn_norm.weight", "offset": 38175047680, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "blk.79.ffn_down.weight", "offset": 38175080448, "shape": [ 28672, 8192 ], "size": 32768, "type": 2 }, { "name": "blk.79.ffn_gate.weight", "offset": 38307201024, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.79.ffn_up.weight", "offset": 38439321600, "shape": [ 8192, 28672 ], "size": 132120576, "type": 2 }, { "name": "blk.79.ffn_norm.weight", "offset": 38571442176, "shape": [ 8192 ], "size": 132120576, "type": 0 }, { "name": "blk.79.attn_k.weight", "offset": 38571474944, "shape": [ 8192, 1024 ], "size": 32768, "type": 2 }, { "name": "blk.79.attn_output.weight", "offset": 38576193536, "shape": [ 8192, 8192 ], "size": 4718592, "type": 2 }, { "name": "blk.79.attn_q.weight", "offset": 38613942272, "shape": [ 8192, 8192 ], "size": 37748736, "type": 2 }, { "name": "blk.79.attn_v.weight", "offset": 38651691008, "shape": [ 8192, 1024 ], "size": 37748736, "type": 2 }, { "name": "output_norm.weight", "offset": 38656409600, "shape": [ 8192 ], "size": 4718592, "type": 0 }, { "name": "output.weight", "offset": 38656442368, "shape": [ 8192, 32000 ], "size": 32768, "type": 14 } ], "version": 3 }