Advancing Open-source Large Language Models in Medical Domain
8B
70B
515 Pulls Updated 5 weeks ago
926050232bc9 · 6.6GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": ".",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 32,
"llama.context_length": 8192,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 14336,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.scores": "... (128256 values)",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 8030261248,
"tensors": [
{
"name": "blk.0.attn_q.weight",
"offset": 430940160,
"shape": [
4096,
4096
],
"size": 430940160,
"type": 14
},
{
"name": "blk.0.attn_k.weight",
"offset": 444702720,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 448143360,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 451584000,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 465346560,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 513515520,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.0.ffn_down.weight",
"offset": 561684480,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 609853440,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 609869824,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 609886208,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.attn_k.weight",
"offset": 623648768,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 627089408,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 630530048,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 644292608,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 692461568,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.1.ffn_down.weight",
"offset": 740630528,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 788799488,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 788815872,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 788832256,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.attn_k.weight",
"offset": 802594816,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 806035456,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 809476096,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 823238656,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 871407616,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.2.ffn_down.weight",
"offset": 919576576,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 967745536,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 967761920,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 967778304,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.attn_k.weight",
"offset": 981540864,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 984981504,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 988422144,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1002184704,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1050353664,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1098522624,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1146691584,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1146707968,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 1146724352,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.attn_k.weight",
"offset": 1160486912,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 1163927552,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 1167368192,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1181130752,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1229299712,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1277468672,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1325637632,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1325654016,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 1325670400,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.attn_k.weight",
"offset": 1339432960,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 1342873600,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 1346314240,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1360076800,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1408245760,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1456414720,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1504583680,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1504600064,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 1504616448,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.attn_k.weight",
"offset": 1518379008,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 1521819648,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 1525260288,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1539022848,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1587191808,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1635360768,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1683529728,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1683546112,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 1683562496,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.attn_k.weight",
"offset": 1697325056,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 1700765696,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 1704206336,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1717968896,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1766137856,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1814306816,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1862475776,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1862492160,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 1862508544,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.attn_k.weight",
"offset": 1876271104,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 1879711744,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 1883152384,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1896914944,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1945083904,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1993252864,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2041421824,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2041438208,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 2041454592,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.attn_k.weight",
"offset": 2055217152,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 2058657792,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 2062098432,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 2075860992,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 2124029952,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.9.ffn_down.weight",
"offset": 2172198912,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 2220367872,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 2220384256,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 2220400640,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.attn_k.weight",
"offset": 2234163200,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 2237603840,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 2241044480,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2254807040,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2302976000,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2351144960,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2399313920,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2399330304,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 2399346688,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.attn_k.weight",
"offset": 2413109248,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 2416549888,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 2419990528,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 2433753088,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 2481922048,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.11.ffn_down.weight",
"offset": 2530091008,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2578259968,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2578276352,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 2578292736,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 2592055296,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 2595495936,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 2598936576,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2612699136,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2660868096,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2709037056,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2757206016,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2757222400,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 2757238784,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.attn_k.weight",
"offset": 2771001344,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 2774441984,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 2777882624,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2791645184,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2839814144,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2887983104,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2936152064,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2936168448,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 2936184832,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.attn_k.weight",
"offset": 2949947392,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 2953388032,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 2956828672,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2970591232,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 3018760192,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.14.ffn_down.weight",
"offset": 3066929152,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 3115098112,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 3115114496,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 3115130880,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.attn_k.weight",
"offset": 3128893440,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.30.ffn_gat",
"offset": 3132334080,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 3135774720,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 3149537280,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 3197706240,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.15.ffn_down.weight",
"offset": 3245875200,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 3294044160,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 3294060544,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 3294076928,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.attn_k.weight",
"offset": 3307839488,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 3311280128,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 3314720768,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 3328483328,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3376652288,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.16.ffn_down.weight",
"offset": 3424821248,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 3472990208,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3473006592,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 3473022976,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.attn_k.weight",
"offset": 3486785536,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 3490226176,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 3493666816,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 3507429376,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 3555598336,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3603767296,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3651936256,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3651952640,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 3651969024,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.attn_k.weight",
"offset": 3665731584,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 3669172224,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 3672612864,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3686375424,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3734544384,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3782713344,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3830882304,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3830898688,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 3830915072,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.attn_k.weight",
"offset": 3844677632,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 3848118272,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 3851558912,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 3865321472,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 3913490432,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.19.ffn_down.weight",
"offset": 3961659392,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 4009828352,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 4009844736,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 4009861120,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.attn_k.weight",
"offset": 4023623680,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 4027064320,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 4030504960,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 4044267520,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 4092436480,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.20.ffn_down.weight",
"offset": 4140605440,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 4188774400,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 4188790784,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 4188807168,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.attn_k.weight",
"offset": 4202569728,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 4206010368,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 4209451008,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 4223213568,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 4271382528,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.21.ffn_down.weight",
"offset": 4319551488,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 4367720448,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 4367736832,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 4367753216,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.attn_k.weight",
"offset": 4381515776,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 4384956416,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 4388397056,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4402159616,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4450328576,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4498497536,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4546666496,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4546682880,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 4546699264,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.attn_k.weight",
"offset": 4560461824,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 4563902464,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 4567343104,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4581105664,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4629274624,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4677443584,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4725612544,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 4725628928,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 4725645312,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.attn_k.weight",
"offset": 4739407872,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 4742848512,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 4746289152,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 4760051712,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 4808220672,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.24.ffn_down.weight",
"offset": 4856389632,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 4904558592,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 4904574976,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 4904591360,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.attn_k.weight",
"offset": 4918353920,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 4921794560,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 4925235200,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 4938997760,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 4987166720,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.25.ffn_down.weight",
"offset": 5035335680,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 5083504640,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 5083521024,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 5083537408,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.attn_k.weight",
"offset": 5097299968,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 5100740608,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 5104181248,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 5117943808,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 5166112768,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.26.ffn_down.weight",
"offset": 5214281728,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 5262450688,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 5262467072,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 5262483456,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.attn_k.weight",
"offset": 5276246016,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 5279686656,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 5283127296,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 5296889856,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 5345058816,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.27.ffn_down.weight",
"offset": 5393227776,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 5441396736,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5441413120,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 5441429504,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.attn_k.weight",
"offset": 5455192064,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 5458632704,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 5462073344,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 5475835904,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 5524004864,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5572173824,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5620342784,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 5620359168,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 5620375552,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.attn_k.weight",
"offset": 5634138112,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 5637578752,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 5641019392,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 5654781952,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 5702950912,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.29.ffn_down.weight",
"offset": 5751119872,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 5799288832,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 5799305216,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 5799321600,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.attn_k.weight",
"offset": 5813084160,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 5816524800,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 5819965440,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "\ufffdw\ufffd\u0007gJ\ufffd\u0005\u000e\ufffd\u001c\ufffd\ufffd\ufffd/(\ufffd$\ufffd\u0010΄",
"offset": 5833728000,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 5881896960,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.30.ffn_down.weight",
"offset": 5930065920,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 5978234880,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 5978251264,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 5978267648,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.attn_k.weight",
"offset": 5992030208,
"shape": [
4096,
1024
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 5995470848,
"shape": [
4096,
1024
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 5998911488,
"shape": [
4096,
4096
],
"size": 3440640,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 6012674048,
"shape": [
4096,
14336
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 6060843008,
"shape": [
4096,
14336
],
"size": 48168960,
"type": 14
},
{
"name": "blk.31.ffn_down.weight",
"offset": 6109011968,
"shape": [
14336,
4096
],
"size": 48168960,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 6157180928,
"shape": [
4096
],
"size": 48168960,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 6157197312,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 6157213696,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "output.weight",
"offset": 6157230080,
"shape": [
4096,
128256
],
"size": 16384,
"type": 14
}
],
"version": 3
}