https://huggingface.co/NTQAI/Nxcode-CQ-7B-orpo
7B
427 Pulls Updated 7 weeks ago
6378a0cac8a8 · 6.4GB
{
"metadata": {
"general.architecture": "qwen2",
"general.file_type": 18,
"general.name": "Nxcode-CQ-7B-orpo",
"general.quantization_version": 2,
"qwen2.attention.head_count": 32,
"qwen2.attention.head_count_kv": 4,
"qwen2.attention.layer_norm_rms_epsilon": 0.00001,
"qwen2.block_count": 32,
"qwen2.context_length": 65536,
"qwen2.embedding_length": 4096,
"qwen2.feed_forward_length": 13440,
"qwen2.rope.freq_base": 1000000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 2,
"tokenizer.ggml.eos_token_id": 4,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 92298,
"tokenizer.ggml.pre": "default",
"tokenizer.ggml.scores": "... (92416 values)",
"tokenizer.ggml.token_type": "... (92416 values)",
"tokenizer.ggml.tokens": "... (92416 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 7250284544,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 310517760,
"shape": [
4096
],
"size": 310517760,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 310534144,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 369025024,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 414183424,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 459341824,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.0.attn_k.bias",
"offset": 459358208,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 459360256,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 461080576,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.0.attn_q.bias",
"offset": 474843136,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.0.attn_q.weight",
"offset": 474859520,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.attn_v.bias",
"offset": 488622080,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.0.attn_v.weight",
"offset": 488624128,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 490344448,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 490360832,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 548851712,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 594010112,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 639168512,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.1.attn_k.bias",
"offset": 639184896,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 639186944,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 640907264,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.1.attn_q.bias",
"offset": 654669824,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 654686208,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.attn_v.bias",
"offset": 668448768,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.1.attn_v.weight",
"offset": 668450816,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 670171136,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 670187520,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 728678400,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 773836800,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 818995200,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.10.attn_k.bias",
"offset": 819011584,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 819013632,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 820733952,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.10.attn_q.bias",
"offset": 834496512,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 834512896,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.attn_v.bias",
"offset": 848275456,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.10.attn_v.weight",
"offset": 848277504,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 849997824,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 850014208,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 908505088,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 953663488,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 998821888,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.11.attn_k.bias",
"offset": 998838272,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 998840320,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 1000560640,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.11.attn_q.bias",
"offset": 1014323200,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 1014339584,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.attn_v.bias",
"offset": 1028102144,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.11.attn_v.weight",
"offset": 1028104192,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1029824512,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1029840896,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 1088331776,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 1133490176,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1178648576,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.12.attn_k.bias",
"offset": 1178664960,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 1178667008,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 1180387328,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.12.attn_q.bias",
"offset": 1194149888,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 1194166272,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.12.attn_v.bias",
"offset": 1207928832,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.12.attn_v.weight",
"offset": 1207930880,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1209651200,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1209667584,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1268158464,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1313316864,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1358475264,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.13.attn_k.bias",
"offset": 1358491648,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "t\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000",
"offset": 1358493696,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 1360214016,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.13.attn_q.bias",
"offset": 1373976576,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 1373992960,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.attn_v.bias",
"offset": 1387755520,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.13.attn_v.weight",
"offset": 1387757568,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1389477888,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1389494272,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1447985152,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1493143552,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1538301952,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.14.attn_k.bias",
"offset": 1538318336,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1538320384,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 1540040704,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.14.attn_q.bias",
"offset": 1553803264,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 1553819648,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.attn_v.bias",
"offset": 1567582208,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.14.attn_v.weight",
"offset": 1567584256,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1569304576,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1569320960,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1627811840,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1672970240,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1718128640,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.15.attn_k.bias",
"offset": 1718145024,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1718147072,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 1719867392,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.15.attn_q.bias",
"offset": 1733629952,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 1733646336,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.attn_v.bias",
"offset": 1747408896,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.15.attn_v.weight",
"offset": 1747410944,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 1749131264,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1749147648,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1807638528,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1852796928,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1897955328,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.16.attn_k.bias",
"offset": 1897971712,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1897973760,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 1899694080,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.16.attn_q.bias",
"offset": 1913456640,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 1913473024,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.attn_v.bias",
"offset": 1927235584,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.16.attn_v.weight",
"offset": 1927237632,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 1928957952,
"shape": [
4096,
13440
],
"size": 1720320,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1974116352,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.17.attn_k.bias",
"offset": 2019274752,
"shape": [
512
],
"size": 45158400,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2019276800,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 2020997120,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.17.attn_q.bias",
"offset": 2034759680,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 2034776064,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.attn_v.bias",
"offset": 2048538624,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.17.attn_v.weight",
"offset": 2048540672,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2050260992,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 2050277376,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2108768256,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2153926656,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2199085056,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.2.attn_k.bias",
"offset": 2199101440,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2199103488,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 2200823808,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.2.attn_q.bias",
"offset": 2214586368,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 2214602752,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.attn_v.bias",
"offset": 2228365312,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.2.attn_v.weight",
"offset": 2228367360,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2230087680,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2230104064,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2288594944,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2333753344,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2378911744,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.3.attn_k.bias",
"offset": 2378928128,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 2378930176,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 2380650496,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.3.attn_q.bias",
"offset": 2394413056,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 2394429440,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.attn_v.bias",
"offset": 2408192000,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.3.attn_v.weight",
"offset": 2408194048,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2409914368,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2409930752,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2468421632,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 2513580032,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 2558738432,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.4.attn_k.bias",
"offset": 2558754816,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 2558756864,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 2560477184,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.4.attn_q.bias",
"offset": 2574239744,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 2574256128,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.attn_v.bias",
"offset": 2588018688,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.4.attn_v.weight",
"offset": 2588020736,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2589741056,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2589757440,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 2648248320,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2693406720,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2738565120,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.5.attn_k.bias",
"offset": 2738581504,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 2738583552,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 2740303872,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.5.attn_q.bias",
"offset": 2754066432,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 2754082816,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.attn_v.bias",
"offset": 2767845376,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.5.attn_v.weight",
"offset": 2767847424,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 2769567744,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 2769584128,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2828075008,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2873233408,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2918391808,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.6.attn_k.bias",
"offset": 2918408192,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2918410240,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 2920130560,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.6.attn_q.bias",
"offset": 2933893120,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 2933909504,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.attn_v.bias",
"offset": 2947672064,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.6.attn_v.weight",
"offset": 2947674112,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2949394432,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2949410816,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 3007901696,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 3053060096,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 3098218496,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.7.attn_k.bias",
"offset": 3098234880,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 3098236928,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 3099957248,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.7.attn_q.bias",
"offset": 3113719808,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 3113736192,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.attn_v.bias",
"offset": 3127498752,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.7.attn_v.weight",
"offset": 3127500800,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 3129221120,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 3129237504,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 3187728384,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 3232886784,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 3278045184,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.8.attn_k.bias",
"offset": 3278061568,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 3278063616,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 3279783936,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.8.attn_q.bias",
"offset": 3293546496,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 3293562880,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.attn_v.bias",
"offset": 3307325440,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.8.attn_v.weight",
"offset": 3307327488,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3309047808,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3309064192,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3367555072,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3412713472,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3457871872,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.9.attn_k.bias",
"offset": 3457888256,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3457890304,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 3459610624,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.9.attn_q.bias",
"offset": 3473373184,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 3473389568,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.attn_v.bias",
"offset": 3487152128,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.9.attn_v.weight",
"offset": 3487154176,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "output.weight",
"offset": 3488874496,
"shape": [
4096,
92416
],
"size": 1720320,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3799392256,
"shape": [
4096
],
"size": 310517760,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3799408640,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3857899520,
"shape": [
4096
],
"size": 58490880,
"type": 0
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3857915904,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3857932288,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3916423168,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3961581568,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 4006739968,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.18.attn_k.bias",
"offset": 4006756352,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 4006758400,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 4008478720,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.18.attn_q.bias",
"offset": 4022241280,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 4022257664,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.attn_v.bias",
"offset": 4036020224,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.18.attn_v.weight",
"offset": 4036022272,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 4037742592,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 4037758976,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 4096249856,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 4141408256,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 4186566656,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.19.attn_k.bias",
"offset": 4186583040,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 4186585088,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 4188305408,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.19.attn_q.bias",
"offset": 4202067968,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 4202084352,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.attn_v.bias",
"offset": 4215846912,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.19.attn_v.weight",
"offset": 4215848960,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 4217569280,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 4217585664,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 4276076544,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 4321234944,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 4366393344,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.20.attn_k.bias",
"offset": 4366409728,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 4366411776,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 4368132096,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.20.attn_q.bias",
"offset": 4381894656,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 4381911040,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.attn_v.bias",
"offset": 4395673600,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.20.attn_v.weight",
"offset": 4395675648,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 4397395968,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 4397412352,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 4455903232,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 4501061632,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 4546220032,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.21.attn_k.bias",
"offset": 4546236416,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 4546238464,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 4547958784,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.21.attn_q.bias",
"offset": 4561721344,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 4561737728,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.attn_v.bias",
"offset": 4575500288,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.21.attn_v.weight",
"offset": 4575502336,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4577222656,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4577239040,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4635729920,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4680888320,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4726046720,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.22.attn_k.bias",
"offset": 4726063104,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 4726065152,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 4727785472,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.22.attn_q.bias",
"offset": 4741548032,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 4741564416,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.attn_v.bias",
"offset": 4755326976,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.22.attn_v.weight",
"offset": 4755329024,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4757049344,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4757065728,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4815556608,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4860715008,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4905873408,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.23.attn_k.bias",
"offset": 4905889792,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 4905891840,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 4907612160,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.23.attn_q.bias",
"offset": 4921374720,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 4921391104,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.attn_v.bias",
"offset": 4935153664,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.23.attn_v.weight",
"offset": 4935155712,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4936876032,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4936892416,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4995383296,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 5040541696,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 5085700096,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.24.attn_k.bias",
"offset": 5085716480,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 5085718528,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 5087438848,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.24.attn_q.bias",
"offset": 5101201408,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 5101217792,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.attn_v.bias",
"offset": 5114980352,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.24.attn_v.weight",
"offset": 5114982400,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 5116702720,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 5116719104,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 5175209984,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 5220368384,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 5265526784,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.25.attn_k.bias",
"offset": 5265543168,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 5265545216,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 5267265536,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.25.attn_q.bias",
"offset": 5281028096,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 5281044480,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.attn_v.bias",
"offset": 5294807040,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.25.attn_v.weight",
"offset": 5294809088,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 5296529408,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 5296545792,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 5355036672,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 5400195072,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 5445353472,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.26.attn_k.bias",
"offset": 5445369856,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 5445371904,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 5447092224,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.26.attn_q.bias",
"offset": 5460854784,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 5460871168,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.attn_v.bias",
"offset": 5474633728,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.26.attn_v.weight",
"offset": 5474635776,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 5476356096,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 5476372480,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 5534863360,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 5580021760,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5625180160,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.27.attn_k.bias",
"offset": 5625196544,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 5625198592,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 5626918912,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.27.attn_q.bias",
"offset": 5640681472,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 5640697856,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.attn_v.bias",
"offset": 5654460416,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.27.attn_v.weight",
"offset": 5654462464,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5656182784,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5656199168,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 5714690048,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 5759848448,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 5805006848,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.28.attn_k.bias",
"offset": 5805023232,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 5805025280,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 5806745600,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.28.attn_q.bias",
"offset": 5820508160,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 5820524544,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.attn_v.bias",
"offset": 5834287104,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.28.attn_v.weight",
"offset": 5834289152,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 5836009472,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 5836025856,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 5894516736,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 5939675136,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 5984833536,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.29.attn_k.bias",
"offset": 5984849920,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 5984851968,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 5986572288,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.29.attn_q.bias",
"offset": 6000334848,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 6000351232,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.attn_v.bias",
"offset": 6014113792,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.29.attn_v.weight",
"offset": 6014115840,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 6015836160,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 6015852544,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 6074343424,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 6119501824,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 6164660224,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.30.attn_k.bias",
"offset": 6164676608,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 6164678656,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 6166398976,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.30.attn_q.bias",
"offset": 6180161536,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 6180177920,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.attn_v.bias",
"offset": 6193940480,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.30.attn_v.weight",
"offset": 6193942528,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 6195662848,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 6195679232,
"shape": [
13440,
4096
],
"size": 16384,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 6254170112,
"shape": [
4096,
13440
],
"size": 58490880,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 6299328512,
"shape": [
4096,
13440
],
"size": 45158400,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 6344486912,
"shape": [
4096
],
"size": 45158400,
"type": 0
},
{
"name": "blk.31.attn_k.bias",
"offset": 6344503296,
"shape": [
512
],
"size": 16384,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 6344505344,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 6346225664,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.31.attn_q.bias",
"offset": 6359988224,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 6360004608,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.attn_v.bias",
"offset": 6373767168,
"shape": [
512
],
"size": 13762560,
"type": 0
},
{
"name": "blk.31.attn_v.weight",
"offset": 6373769216,
"shape": [
4096,
512
],
"size": 2048,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 6375489536,
"shape": [
4096
],
"size": 1720320,
"type": 0
}
],
"version": 3
}