Advancing Open-source Large Language Models in Medical Domain , in Chinese or mutil-languages.
70B
270 Pulls Updated 5 weeks ago
47c4b6e698bb · 58GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": ".",
"general.quantization_version": 2,
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 8192,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.scores": "... (128256 values)",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "blk.0.attn_q.weight",
"offset": 861880320,
"shape": [
8192,
8192
],
"size": 861880320,
"type": 14
},
{
"name": "blk.0.attn_k.weight",
"offset": 916930560,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 923811840,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 930693120,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 985743360,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1178419200,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1371095040,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1563770880,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1563803648,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 1563836416,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.1.attn_k.weight",
"offset": 1618886656,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 1625767936,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 1632649216,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1687699456,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1880375296,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.1.ffn_down.weight",
"offset": 2073051136,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2265726976,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2265759744,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 2265792512,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.2.attn_k.weight",
"offset": 2320842752,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 2327724032,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 2334605312,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 2389655552,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 2582331392,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.2.ffn_down.weight",
"offset": 2775007232,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2967683072,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2967715840,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 2967748608,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.3.attn_k.weight",
"offset": 3022798848,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 3029680128,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 3036561408,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 3091611648,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 3284287488,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.3.ffn_down.weight",
"offset": 3476963328,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 3669639168,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 3669671936,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 3669704704,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.4.attn_k.weight",
"offset": 3724754944,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 3731636224,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 3738517504,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 3793567744,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 3986243584,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.4.ffn_down.weight",
"offset": 4178919424,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 4371595264,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 4371628032,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 4371660800,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.5.attn_k.weight",
"offset": 4426711040,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 4433592320,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 4440473600,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 4495523840,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 4688199680,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.5.ffn_down.weight",
"offset": 4880875520,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 5073551360,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 5073584128,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 5073616896,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.6.attn_k.weight",
"offset": 5128667136,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 5135548416,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 5142429696,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 5197479936,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 5390155776,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.6.ffn_down.weight",
"offset": 5582831616,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 5775507456,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 5775540224,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 5775572992,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.7.attn_k.weight",
"offset": 5830623232,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 5837504512,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 5844385792,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 5899436032,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 6092111872,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.7.ffn_down.weight",
"offset": 6284787712,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 6477463552,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 6477496320,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 6477529088,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.8.attn_k.weight",
"offset": 6532579328,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 6539460608,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 6546341888,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 6601392128,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 6794067968,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.8.ffn_down.weight",
"offset": 6986743808,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 7179419648,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 7179452416,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 7179485184,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.9.attn_k.weight",
"offset": 7234535424,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 7241416704,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 7248297984,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 7303348224,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 7496024064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.9.ffn_down.weight",
"offset": 7688699904,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 7881375744,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 7881408512,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 7881441280,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.10.attn_k.weight",
"offset": 7936491520,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 7943372800,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 7950254080,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 8005304320,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 8197980160,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.10.ffn_down.weight",
"offset": 8390656000,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 8583331840,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 8583364608,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 8583397376,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.11.attn_k.weight",
"offset": 8638447616,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 8645328896,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 8652210176,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 8707260416,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 8899936256,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.11.ffn_down.weight",
"offset": 9092612096,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 9285287936,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 9285320704,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 9285353472,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 9340403712,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 9347284992,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 9354166272,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 9409216512,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 9601892352,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.12.ffn_down.weight",
"offset": 9794568192,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 9987244032,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 9987276800,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 9987309568,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.13.attn_k.weight",
"offset": 10042359808,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 10049241088,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 10056122368,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 10111172608,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 10303848448,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.13.ffn_down.weight",
"offset": 10496524288,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 10689200128,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 10689232896,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 10689265664,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.14.attn_k.weight",
"offset": 10744315904,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 10751197184,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 10758078464,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 10813128704,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 11005804544,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.14.ffn_down.weight",
"offset": 11198480384,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 11391156224,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 11391188992,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 11391221760,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.15.attn_k.weight",
"offset": 11446272000,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.15.attn_v.weight",
"offset": 11453153280,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 11460034560,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 11515084800,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 11707760640,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.15.ffn_down.weight",
"offset": 11900436480,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 12093112320,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 12093145088,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 12093177856,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.16.attn_k.weight",
"offset": 12148228096,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 12155109376,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 12161990656,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 12217040896,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 12409716736,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.16.ffn_down.weight",
"offset": 12602392576,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 12795068416,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 12795101184,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 12795133952,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.17.attn_k.weight",
"offset": 12850184192,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 12857065472,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 12863946752,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 12918996992,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 13111672832,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.17.ffn_down.weight",
"offset": 13304348672,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 13497024512,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 13497057280,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 13497090048,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.18.attn_k.weight",
"offset": 13552140288,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 13559021568,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 13565902848,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 13620953088,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 13813628928,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.18.ffn_down.weight",
"offset": 14006304768,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 14198980608,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 14199013376,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 14199046144,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.19.attn_k.weight",
"offset": 14254096384,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 14260977664,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 14267858944,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 14322909184,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 14515585024,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.19.ffn_down.weight",
"offset": 14708260864,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 14900936704,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 14900969472,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 14901002240,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.20.attn_k.weight",
"offset": 14956052480,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 14962933760,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 14969815040,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 15024865280,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 15217541120,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.20.ffn_down.weight",
"offset": 15410216960,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 15602892800,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 15602925568,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 15602958336,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.21.attn_k.weight",
"offset": 15658008576,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 15664889856,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 15671771136,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 15726821376,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 15919497216,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.21.ffn_down.weight",
"offset": 16112173056,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 16304848896,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 16304881664,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 16304914432,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.22.attn_k.weight",
"offset": 16359964672,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 16366845952,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 16373727232,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 16428777472,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 16621453312,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.22.ffn_down.weight",
"offset": 16814129152,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 17006804992,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 17006837760,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 17006870528,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.23.attn_k.weight",
"offset": 17061920768,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 17068802048,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 17075683328,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 17130733568,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 17323409408,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.23.ffn_down.weight",
"offset": 17516085248,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 17708761088,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 17708793856,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 17708826624,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.24.attn_k.weight",
"offset": 17763876864,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 17770758144,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 17777639424,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 17832689664,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 18025365504,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.24.ffn_down.weight",
"offset": 18218041344,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 18410717184,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 18410749952,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 18410782720,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.25.attn_k.weight",
"offset": 18465832960,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 18472714240,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 18479595520,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 18534645760,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 18727321600,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.25.ffn_down.weight",
"offset": 18919997440,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 19112673280,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 19112706048,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 19112738816,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.26.attn_k.weight",
"offset": 19167789056,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 19174670336,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 19181551616,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 19236601856,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 19429277696,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.26.ffn_down.weight",
"offset": 19621953536,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 19814629376,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 19814662144,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 19814694912,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.27.attn_k.weight",
"offset": 19869745152,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 19876626432,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 19883507712,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 19938557952,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 20131233792,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.27.ffn_down.weight",
"offset": 20323909632,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 20516585472,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 20516618240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 20516651008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.28.attn_k.weight",
"offset": 20571701248,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 20578582528,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 20585463808,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 20640514048,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 20833189888,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.28.ffn_down.weight",
"offset": 21025865728,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 21218541568,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 21218574336,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 21218607104,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.29.attn_k.weight",
"offset": 21273657344,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 21280538624,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 21287419904,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 21342470144,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 21535145984,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.29.ffn_down.weight",
"offset": 21727821824,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 21920497664,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 21920530432,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 21920563200,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.30.attn_k.weight",
"offset": 21975613440,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 21982494720,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 21989376000,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 22044426240,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 22237102080,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.30.ffn_down.weight",
"offset": 22429777920,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 22622453760,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 22622486528,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 22622519296,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.31.attn_k.weight",
"offset": 22677569536,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 22684450816,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 22691332096,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 22746382336,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 22939058176,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.31.ffn_down.weight",
"offset": 23131734016,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 23324409856,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 23324442624,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.32.attn_q.weight",
"offset": 23324475392,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.32.attn_k.weight",
"offset": 23379525632,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.32.attn_v.weight",
"offset": 23386406912,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.32.attn_output.weight",
"offset": 23393288192,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 23448338432,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.32.ffn_up.weight",
"offset": 23641014272,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.32.ffn_down.weight",
"offset": 23833690112,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.32.attn_norm.weight",
"offset": 24026365952,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 24026398720,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.33.attn_q.weight",
"offset": 24026431488,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.33.attn_k.weight",
"offset": 24081481728,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.33.attn_v.weight",
"offset": 24088363008,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.33.attn_output.weight",
"offset": 24095244288,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 24150294528,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.33.ffn_up.weight",
"offset": 24342970368,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.33.ffn_down.weight",
"offset": 24535646208,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 24728322048,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 24728354816,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.34.attn_q.weight",
"offset": 24728387584,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.34.attn_k.weight",
"offset": 24783437824,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.34.attn_v.weight",
"offset": 24790319104,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.34.attn_output.weight",
"offset": 24797200384,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 24852250624,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.34.ffn_up.weight",
"offset": 25044926464,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.34.ffn_down.weight",
"offset": 25237602304,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.34.attn_norm.weight",
"offset": 25430278144,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 25430310912,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.35.attn_q.weight",
"offset": 25430343680,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.35.attn_k.weight",
"offset": 25485393920,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.35.attn_v.weight",
"offset": 25492275200,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.35.attn_output.weight",
"offset": 25499156480,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 25554206720,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.35.ffn_up.weight",
"offset": 25746882560,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.35.ffn_down.weight",
"offset": 25939558400,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.35.attn_norm.weight",
"offset": 26132234240,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 26132267008,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.36.attn_q.weight",
"offset": 26132299776,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.36.attn_k.weight",
"offset": 26187350016,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.36.attn_v.weight",
"offset": 26194231296,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.36.attn_output.weight",
"offset": 26201112576,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 26256162816,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.36.ffn_up.weight",
"offset": 26448838656,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.36.ffn_down.weight",
"offset": 26641514496,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 26834190336,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 26834223104,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.37.attn_q.weight",
"offset": 26834255872,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.37.attn_k.weight",
"offset": 26889306112,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.37.attn_v.weight",
"offset": 26896187392,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.37.attn_output.weight",
"offset": 26903068672,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 26958118912,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.37.ffn_up.weight",
"offset": 27150794752,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.37.ffn_down.weight",
"offset": 27343470592,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 27536146432,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 27536179200,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.38.attn_q.weight",
"offset": 27536211968,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.38.attn_k.weight",
"offset": 27591262208,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.38.attn_v.weight",
"offset": 27598143488,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.38.attn_output.weight",
"offset": 27605024768,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 27660075008,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.38.ffn_up.weight",
"offset": 27852750848,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.38.ffn_down.weight",
"offset": 28045426688,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.38.attn_norm.weight",
"offset": 28238102528,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 28238135296,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.39.attn_q.weight",
"offset": 28238168064,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.39.attn_k.weight",
"offset": 28293218304,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.39.attn_v.weight",
"offset": 28300099584,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.39.attn_output.weight",
"offset": 28306980864,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 28362031104,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.39.ffn_up.weight",
"offset": 28554706944,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.39.ffn_down.weight",
"offset": 28747382784,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 28940058624,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 28940091392,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.40.attn_q.weight",
"offset": 28940124160,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.40.attn_k.weight",
"offset": 28995174400,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.40.attn_v.weight",
"offset": 29002055680,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.40.attn_output.weight",
"offset": 29008936960,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 29063987200,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.40.ffn_up.weight",
"offset": 29256663040,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.40.ffn_down.weight",
"offset": 29449338880,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 29642014720,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 29642047488,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.41.attn_q.weight",
"offset": 29642080256,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.41.attn_k.weight",
"offset": 29697130496,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.41.attn_v.weight",
"offset": 29704011776,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.41.attn_output.weight",
"offset": 29710893056,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 29765943296,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 29958619136,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.41.ffn_down.weight",
"offset": 30151294976,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 30343970816,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 30344003584,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.42.attn_q.weight",
"offset": 30344036352,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.42.attn_k.weight",
"offset": 30399086592,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.42.attn_v.weight",
"offset": 30405967872,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.42.attn_output.weight",
"offset": 30412849152,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 30467899392,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 30660575232,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.42.ffn_down.weight",
"offset": 30853251072,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 31045926912,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 31045959680,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.43.attn_q.weight",
"offset": 31045992448,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.43.attn_k.weight",
"offset": 31101042688,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.43.attn_v.weight",
"offset": 31107923968,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.43.attn_output.weight",
"offset": 31114805248,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 31169855488,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.43.ffn_up.weight",
"offset": 31362531328,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.43.ffn_down.weight",
"offset": 31555207168,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.43.attn_norm.weight",
"offset": 31747883008,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 31747915776,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.44.attn_q.weight",
"offset": 31747948544,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.44.attn_k.weight",
"offset": 31802998784,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.44.attn_v.weight",
"offset": 31809880064,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.44.attn_output.weight",
"offset": 31816761344,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 31871811584,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.44.ffn_up.weight",
"offset": 32064487424,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.44.ffn_down.weight",
"offset": 32257163264,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 32449839104,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 32449871872,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.45.attn_q.weight",
"offset": 32449904640,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.45.attn_k.weight",
"offset": 32504954880,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.45.attn_v.weight",
"offset": 32511836160,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.45.attn_output.weight",
"offset": 32518717440,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 32573767680,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.45.ffn_up.weight",
"offset": 32766443520,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.45.ffn_down.weight",
"offset": 32959119360,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 33151795200,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 33151827968,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.46.attn_q.weight",
"offset": 33151860736,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.46.attn_k.weight",
"offset": 33206910976,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.46.attn_v.weight",
"offset": 33213792256,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.46.attn_output.weight",
"offset": 33220673536,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 33275723776,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.46.ffn_up.weight",
"offset": 33468399616,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.46.ffn_down.weight",
"offset": 33661075456,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 33853751296,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 33853784064,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.47.attn_q.weight",
"offset": 33853816832,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.47.attn_k.weight",
"offset": 33908867072,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.47.attn_v.weight",
"offset": 33915748352,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.47.attn_output.weight",
"offset": 33922629632,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 33977679872,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.47.ffn_up.weight",
"offset": 34170355712,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.47.ffn_down.weight",
"offset": 34363031552,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.47.attn_norm.weight",
"offset": 34555707392,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 34555740160,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.48.attn_q.weight",
"offset": 34555772928,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.48.attn_k.weight",
"offset": 34610823168,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.48.attn_v.weight",
"offset": 34617704448,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.48.attn_output.weight",
"offset": 34624585728,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 34679635968,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.48.ffn_up.weight",
"offset": 34872311808,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.48.ffn_down.weight",
"offset": 35064987648,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 35257663488,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 35257696256,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.49.attn_q.weight",
"offset": 35257729024,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.49.attn_k.weight",
"offset": 35312779264,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.49.attn_v.weight",
"offset": 35319660544,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.49.attn_output.weight",
"offset": 35326541824,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 35381592064,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.49.ffn_up.weight",
"offset": 35574267904,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.49.ffn_down.weight",
"offset": 35766943744,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.49.attn_norm.weight",
"offset": 35959619584,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 35959652352,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.50.attn_q.weight",
"offset": 35959685120,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.50.attn_k.weight",
"offset": 36014735360,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.50.attn_v.weight",
"offset": 36021616640,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.50.attn_output.weight",
"offset": 36028497920,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 36083548160,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.50.ffn_up.weight",
"offset": 36276224000,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.50.ffn_down.weight",
"offset": 36468899840,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.50.attn_norm.weight",
"offset": 36661575680,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 36661608448,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.51.attn_q.weight",
"offset": 36661641216,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.51.attn_k.weight",
"offset": 36716691456,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.51.attn_v.weight",
"offset": 36723572736,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.51.attn_output.weight",
"offset": 36730454016,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 36785504256,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.51.ffn_up.weight",
"offset": 36978180096,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.51.ffn_down.weight",
"offset": 37170855936,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 37363531776,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 37363564544,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.52.attn_q.weight",
"offset": 37363597312,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.52.attn_k.weight",
"offset": 37418647552,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.52.attn_v.weight",
"offset": 37425528832,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.52.attn_output.weight",
"offset": 37432410112,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 37487460352,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.52.ffn_up.weight",
"offset": 37680136192,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.52.ffn_down.weight",
"offset": 37872812032,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.52.attn_norm.weight",
"offset": 38065487872,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 38065520640,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.53.attn_q.weight",
"offset": 38065553408,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.53.attn_k.weight",
"offset": 38120603648,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.53.attn_v.weight",
"offset": 38127484928,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.53.attn_output.weight",
"offset": 38134366208,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 38189416448,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.53.ffn_up.weight",
"offset": 38382092288,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.53.ffn_down.weight",
"offset": 38574768128,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 38767443968,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 38767476736,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.54.attn_q.weight",
"offset": 38767509504,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.54.attn_k.weight",
"offset": 38822559744,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.54.attn_v.weight",
"offset": 38829441024,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.54.attn_output.weight",
"offset": 38836322304,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 38891372544,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.54.ffn_up.weight",
"offset": 39084048384,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.54.ffn_down.weight",
"offset": 39276724224,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 39469400064,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 39469432832,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.55.attn_q.weight",
"offset": 39469465600,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.55.attn_k.weight",
"offset": 39524515840,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.55.attn_v.weight",
"offset": 39531397120,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.55.attn_output.weight",
"offset": 39538278400,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 39593328640,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.55.ffn_up.weight",
"offset": 39786004480,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.55.ffn_down.weight",
"offset": 39978680320,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.55.attn_norm.weight",
"offset": 40171356160,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 40171388928,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.56.attn_q.weight",
"offset": 40171421696,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.56.attn_k.weight",
"offset": 40226471936,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.56.attn_v.weight",
"offset": 40233353216,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.56.attn_output.weight",
"offset": 40240234496,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 40295284736,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.56.ffn_up.weight",
"offset": 40487960576,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.56.ffn_down.weight",
"offset": 40680636416,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.56.attn_norm.weight",
"offset": 40873312256,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 40873345024,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.57.attn_q.weight",
"offset": 40873377792,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.57.attn_k.weight",
"offset": 40928428032,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.57.attn_v.weight",
"offset": 40935309312,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.57.attn_output.weight",
"offset": 40942190592,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 40997240832,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.57.ffn_up.weight",
"offset": 41189916672,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.57.ffn_down.weight",
"offset": 41382592512,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 41575268352,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 41575301120,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.58.attn_q.weight",
"offset": 41575333888,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.58.attn_k.weight",
"offset": 41630384128,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.58.attn_v.weight",
"offset": 41637265408,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.58.attn_output.weight",
"offset": 41644146688,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 41699196928,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.58.ffn_up.weight",
"offset": 41891872768,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.58.ffn_down.weight",
"offset": 42084548608,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.58.attn_norm.weight",
"offset": 42277224448,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 42277257216,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.59.attn_q.weight",
"offset": 42277289984,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.59.attn_k.weight",
"offset": 42332340224,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.59.attn_v.weight",
"offset": 42339221504,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.59.attn_output.weight",
"offset": 42346102784,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 42401153024,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.59.ffn_up.weight",
"offset": 42593828864,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.59.ffn_down.weight",
"offset": 42786504704,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 42979180544,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 42979213312,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.60.attn_q.weight",
"offset": 42979246080,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "\u000e\u0000\u0000\u0000\u0000@t\u0005\n\u0000\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000",
"offset": 43034296320,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.60.attn_v.weight",
"offset": 43041177600,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.60.attn_output.weight",
"offset": 43048058880,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 43103109120,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.60.ffn_up.weight",
"offset": 43295784960,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.60.ffn_down.weight",
"offset": 43488460800,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.60.attn_norm.weight",
"offset": 43681136640,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 43681169408,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.61.attn_q.weight",
"offset": 43681202176,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.61.attn_k.weight",
"offset": 43736252416,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.61.attn_v.weight",
"offset": 43743133696,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.61.attn_output.weight",
"offset": 43750014976,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 43805065216,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.61.ffn_up.weight",
"offset": 43997741056,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.61.ffn_down.weight",
"offset": 44190416896,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.61.attn_norm.weight",
"offset": 44383092736,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 44383125504,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.62.attn_q.weight",
"offset": 44383158272,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.62.attn_k.weight",
"offset": 44438208512,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.62.attn_v.weight",
"offset": 44445089792,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.62.attn_output.weight",
"offset": 44451971072,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 44507021312,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.62.ffn_up.weight",
"offset": 44699697152,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.62.ffn_down.weight",
"offset": 44892372992,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.62.attn_norm.weight",
"offset": 45085048832,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 45085081600,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.63.attn_q.weight",
"offset": 45085114368,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.63.attn_k.weight",
"offset": 45140164608,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.63.attn_v.weight",
"offset": 45147045888,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.63.attn_output.weight",
"offset": 45153927168,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 45208977408,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.63.ffn_up.weight",
"offset": 45401653248,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.63.ffn_down.weight",
"offset": 45594329088,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.63.attn_norm.weight",
"offset": 45787004928,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 45787037696,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.64.attn_q.weight",
"offset": 45787070464,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.64.attn_k.weight",
"offset": 45842120704,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.64.attn_v.weight",
"offset": 45849001984,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.64.attn_output.weight",
"offset": 45855883264,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 45910933504,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.64.ffn_up.weight",
"offset": 46103609344,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.64.ffn_down.weight",
"offset": 46296285184,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.64.attn_norm.weight",
"offset": 46488961024,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 46488993792,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.65.attn_q.weight",
"offset": 46489026560,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.65.attn_k.weight",
"offset": 46544076800,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.65.attn_v.weight",
"offset": 46550958080,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.65.attn_output.weight",
"offset": 46557839360,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 46612889600,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.65.ffn_up.weight",
"offset": 46805565440,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.65.ffn_down.weight",
"offset": 46998241280,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.65.attn_norm.weight",
"offset": 47190917120,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 47190949888,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.66.attn_q.weight",
"offset": 47190982656,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.66.attn_k.weight",
"offset": 47246032896,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.66.attn_v.weight",
"offset": 47252914176,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.66.attn_output.weight",
"offset": 47259795456,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 47314845696,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.66.ffn_up.weight",
"offset": 47507521536,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.66.ffn_down.weight",
"offset": 47700197376,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.66.attn_norm.weight",
"offset": 47892873216,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 47892905984,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.67.attn_q.weight",
"offset": 47892938752,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.67.attn_k.weight",
"offset": 47947988992,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.67.attn_v.weight",
"offset": 47954870272,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.67.attn_output.weight",
"offset": 47961751552,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 48016801792,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.67.ffn_up.weight",
"offset": 48209477632,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.67.ffn_down.weight",
"offset": 48402153472,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.67.attn_norm.weight",
"offset": 48594829312,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 48594862080,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.68.attn_q.weight",
"offset": 48594894848,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.68.attn_k.weight",
"offset": 48649945088,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.68.attn_v.weight",
"offset": 48656826368,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.68.attn_output.weight",
"offset": 48663707648,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 48718757888,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.68.ffn_up.weight",
"offset": 48911433728,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.68.ffn_down.weight",
"offset": 49104109568,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.68.attn_norm.weight",
"offset": 49296785408,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 49296818176,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.69.attn_q.weight",
"offset": 49296850944,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.69.attn_k.weight",
"offset": 49351901184,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.69.attn_v.weight",
"offset": 49358782464,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.69.attn_output.weight",
"offset": 49365663744,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 49420713984,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.69.ffn_up.weight",
"offset": 49613389824,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.69.ffn_down.weight",
"offset": 49806065664,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.69.attn_norm.weight",
"offset": 49998741504,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 49998774272,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.70.attn_q.weight",
"offset": 49998807040,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.70.attn_k.weight",
"offset": 50053857280,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.70.attn_v.weight",
"offset": 50060738560,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.70.attn_output.weight",
"offset": 50067619840,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 50122670080,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.70.ffn_up.weight",
"offset": 50315345920,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.70.ffn_down.weight",
"offset": 50508021760,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.70.attn_norm.weight",
"offset": 50700697600,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 50700730368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.71.attn_q.weight",
"offset": 50700763136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.71.attn_k.weight",
"offset": 50755813376,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.71.attn_v.weight",
"offset": 50762694656,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.71.attn_output.weight",
"offset": 50769575936,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 50824626176,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.71.ffn_up.weight",
"offset": 51017302016,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.71.ffn_down.weight",
"offset": 51209977856,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.71.attn_norm.weight",
"offset": 51402653696,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 51402686464,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.72.attn_q.weight",
"offset": 51402719232,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.72.attn_k.weight",
"offset": 51457769472,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.72.attn_v.weight",
"offset": 51464650752,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.72.attn_output.weight",
"offset": 51471532032,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 51526582272,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.72.ffn_up.weight",
"offset": 51719258112,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.72.ffn_down.weight",
"offset": 51911933952,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.72.attn_norm.weight",
"offset": 52104609792,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 52104642560,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.73.attn_q.weight",
"offset": 52104675328,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.73.attn_k.weight",
"offset": 52159725568,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.73.attn_v.weight",
"offset": 52166606848,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.73.attn_output.weight",
"offset": 52173488128,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 52228538368,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.73.ffn_up.weight",
"offset": 52421214208,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.73.ffn_down.weight",
"offset": 52613890048,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.73.attn_norm.weight",
"offset": 52806565888,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 52806598656,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.74.attn_q.weight",
"offset": 52806631424,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.74.attn_k.weight",
"offset": 52861681664,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.74.attn_v.weight",
"offset": 52868562944,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.74.attn_output.weight",
"offset": 52875444224,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 52930494464,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.74.ffn_up.weight",
"offset": 53123170304,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.74.ffn_down.weight",
"offset": 53315846144,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.74.attn_norm.weight",
"offset": 53508521984,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 53508554752,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.75.attn_q.weight",
"offset": 53508587520,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.75.attn_k.weight",
"offset": 53563637760,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.75.attn_v.weight",
"offset": 53570519040,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "9\ufffd\u0004\ufffd\u001c\ufffd\ufffd\ufffd\u001c#\ufffd\u0013ǣ\ufffdac\ufffd\ufffdƎr\u00101G",
"offset": 53577400320,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 53632450560,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.75.ffn_up.weight",
"offset": 53825126400,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.75.ffn_down.weight",
"offset": 54017802240,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.75.attn_norm.weight",
"offset": 54210478080,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 54210510848,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.76.attn_q.weight",
"offset": 54210543616,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.76.attn_k.weight",
"offset": 54265593856,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.76.attn_v.weight",
"offset": 54272475136,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.76.attn_output.weight",
"offset": 54279356416,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 54334406656,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.76.ffn_up.weight",
"offset": 54527082496,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.76.ffn_down.weight",
"offset": 54719758336,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.76.attn_norm.weight",
"offset": 54912434176,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 54912466944,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.77.attn_q.weight",
"offset": 54912499712,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.77.attn_k.weight",
"offset": 54967549952,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.77.attn_v.weight",
"offset": 54974431232,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.77.attn_output.weight",
"offset": 54981312512,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 55036362752,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.77.ffn_up.weight",
"offset": 55229038592,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.77.ffn_down.weight",
"offset": 55421714432,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.77.attn_norm.weight",
"offset": 55614390272,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 55614423040,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.78.attn_q.weight",
"offset": 55614455808,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.78.attn_k.weight",
"offset": 55669506048,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.78.attn_v.weight",
"offset": 55676387328,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.78.attn_output.weight",
"offset": 55683268608,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 55738318848,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.78.ffn_up.weight",
"offset": 55930994688,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.78.ffn_down.weight",
"offset": 56123670528,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.78.attn_norm.weight",
"offset": 56316346368,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 56316379136,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.79.attn_q.weight",
"offset": 56316411904,
"shape": [
8192,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.79.attn_k.weight",
"offset": 56371462144,
"shape": [
8192,
1024
],
"size": 55050240,
"type": 14
},
{
"name": "blk.79.attn_v.weight",
"offset": 56378343424,
"shape": [
8192,
1024
],
"size": 6881280,
"type": 14
},
{
"name": "blk.79.attn_output.weight",
"offset": 56385224704,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 14
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 56440274944,
"shape": [
8192,
28672
],
"size": 55050240,
"type": 14
},
{
"name": "blk.79.ffn_up.weight",
"offset": 56632950784,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 14
},
{
"name": "blk.79.ffn_down.weight",
"offset": 56825626624,
"shape": [
28672,
8192
],
"size": 192675840,
"type": 14
},
{
"name": "blk.79.attn_norm.weight",
"offset": 57018302464,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 57018335232,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 57018368000,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output.weight",
"offset": 57018400768,
"shape": [
8192,
128256
],
"size": 32768,
"type": 14
}
],
"version": 3
}