latest
80GB
The Mixtral-8x22B Large Language Model (LLM) is a pretrained generative Sparse Mixture of Experts.
8x22B
309 Pulls Updated 2 months ago
47e8af9870a2 · 80GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 2,
"general.name": "models",
"general.quantization_version": 2,
"llama.attention.head_count": 48,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 56,
"llama.context_length": 65536,
"llama.embedding_length": 6144,
"llama.expert_count": 8,
"llama.expert_used_count": 2,
"llama.feed_forward_length": 16384,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 1000000,
"tokenizer.ggml.add_bos_token": true,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 2,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.scores": "... (32000 values)",
"tokenizer.ggml.token_type": "... (32000 values)",
"tokenizer.ggml.tokens": "... (32000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 140620634112,
"tensors": [
{
"name": "blk.0.ffn_gate.0.weight",
"offset": 110592000,
"shape": [
6144,
16384
],
"size": 110592000,
"type": 2
},
{
"name": "blk.0.ffn_down.0.weight",
"offset": 167215104,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.0.weight",
"offset": 223838208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.1.weight",
"offset": 280461312,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.1.weight",
"offset": 337084416,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.1.weight",
"offset": 393707520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.2.weight",
"offset": 450330624,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.2.weight",
"offset": 506953728,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.2.weight",
"offset": 563576832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.3.weight",
"offset": 620199936,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.3.weight",
"offset": 676823040,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.3.weight",
"offset": 733446144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.4.weight",
"offset": 790069248,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.4.weight",
"offset": 846692352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.4.weight",
"offset": 903315456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.5.weight",
"offset": 959938560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.5.weight",
"offset": 1016561664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.5.weight",
"offset": 1073184768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.6.weight",
"offset": 1129807872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_down.6.weight",
"offset": 1186430976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_up.6.weight",
"offset": 1243054080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate.7.weight",
"offset": 1299677184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 1356300288,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.0.attn_k.weight",
"offset": 1356398592,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 1363083264,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.0.attn_q.weight",
"offset": 1384316928,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.0.attn_v.weight",
"offset": 1405550592,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.0.ffn_down.7.weight",
"offset": 1412235264,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.0.ffn_up.7.weight",
"offset": 1468858368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1525481472,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1525506048,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.1.ffn_gate.0.weight",
"offset": 1525530624,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.1.ffn_down.0.weight",
"offset": 1582153728,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.0.weight",
"offset": 1638776832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.1.weight",
"offset": 1695399936,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.1.weight",
"offset": 1752023040,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.1.weight",
"offset": 1808646144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.2.weight",
"offset": 1865269248,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.2.weight",
"offset": 1921892352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.2.weight",
"offset": 1978515456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.3.weight",
"offset": 2035138560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.3.weight",
"offset": 2091761664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.3.weight",
"offset": 2148384768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.4.weight",
"offset": 2205007872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.4.weight",
"offset": 2261630976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.4.weight",
"offset": 2318254080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.5.weight",
"offset": 2374877184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.5.weight",
"offset": 2431500288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.5.weight",
"offset": 2488123392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate.6.weight",
"offset": 2544746496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_down.6.weight",
"offset": 2601369600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.6.weight",
"offset": 2657992704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 2714615808,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.1.attn_k.weight",
"offset": 2714714112,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 2721398784,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.1.attn_q.weight",
"offset": 2742632448,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.1.attn_v.weight",
"offset": 2763866112,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.1.ffn_gate.7.weight",
"offset": 2770550784,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.1.ffn_down.7.weight",
"offset": 2827173888,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.ffn_up.7.weight",
"offset": 2883796992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2940420096,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2940444672,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.2.ffn_gate.0.weight",
"offset": 2940469248,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.2.ffn_down.0.weight",
"offset": 2997092352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.0.weight",
"offset": 3053715456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.1.weight",
"offset": 3110338560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.1.weight",
"offset": 3166961664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.1.weight",
"offset": 3223584768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.2.weight",
"offset": 3280207872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.2.weight",
"offset": 3336830976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.2.weight",
"offset": 3393454080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.3.weight",
"offset": 3450077184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.3.weight",
"offset": 3506700288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.3.weight",
"offset": 3563323392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.4.weight",
"offset": 3619946496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.4.weight",
"offset": 3676569600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.4.weight",
"offset": 3733192704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.5.weight",
"offset": 3789815808,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.5.weight",
"offset": 3846438912,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.5.weight",
"offset": 3903062016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate.6.weight",
"offset": 3959685120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.6.weight",
"offset": 4016308224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 4072931328,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.2.attn_k.weight",
"offset": 4073029632,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 4079714304,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.2.attn_q.weight",
"offset": 4100947968,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.2.attn_v.weight",
"offset": 4122181632,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.2.ffn_up.6.weight",
"offset": 4128866304,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.2.ffn_gate.7.weight",
"offset": 4185489408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_down.7.weight",
"offset": 4242112512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.ffn_up.7.weight",
"offset": 4298735616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.2.attn_norm.weight",
"offset": 4355358720,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 4355383296,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.3.ffn_gate.0.weight",
"offset": 4355407872,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.3.ffn_down.0.weight",
"offset": 4412030976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.0.weight",
"offset": 4468654080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.1.weight",
"offset": 4525277184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.1.weight",
"offset": 4581900288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.1.weight",
"offset": 4638523392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.2.weight",
"offset": 4695146496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.2.weight",
"offset": 4751769600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.2.weight",
"offset": 4808392704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.3.weight",
"offset": 4865015808,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.3.weight",
"offset": 4921638912,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.3.weight",
"offset": 4978262016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.4.weight",
"offset": 5034885120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.4.weight",
"offset": 5091508224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.4.weight",
"offset": 5148131328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.5.weight",
"offset": 5204754432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.5.weight",
"offset": 5261377536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.5.weight",
"offset": 5318000640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.6.weight",
"offset": 5374623744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 5431246848,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.3.attn_k.weight",
"offset": 5431345152,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 5438029824,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.3.attn_q.weight",
"offset": 5459263488,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.3.attn_v.weight",
"offset": 5480497152,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.3.ffn_down.6.weight",
"offset": 5487181824,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.3.ffn_up.6.weight",
"offset": 5543804928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_gate.7.weight",
"offset": 5600428032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_down.7.weight",
"offset": 5657051136,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.ffn_up.7.weight",
"offset": 5713674240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.3.attn_norm.weight",
"offset": 5770297344,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 5770321920,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.4.ffn_gate.0.weight",
"offset": 5770346496,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.4.ffn_down.0.weight",
"offset": 5826969600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.0.weight",
"offset": 5883592704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.1.weight",
"offset": 5940215808,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.1.weight",
"offset": 5996838912,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.1.weight",
"offset": 6053462016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.2.weight",
"offset": 6110085120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.2.weight",
"offset": 6166708224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.2.weight",
"offset": 6223331328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.3.weight",
"offset": 6279954432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.3.weight",
"offset": 6336577536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.3.weight",
"offset": 6393200640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.4.weight",
"offset": 6449823744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.4.weight",
"offset": 6506446848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.4.weight",
"offset": 6563069952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.5.weight",
"offset": 6619693056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.5.weight",
"offset": 6676316160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.5.weight",
"offset": 6732939264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 6789562368,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.4.attn_k.weight",
"offset": 6789660672,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 6796345344,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.4.attn_q.weight",
"offset": 6817579008,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.4.attn_v.weight",
"offset": 6838812672,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.4.ffn_gate.6.weight",
"offset": 6845497344,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.4.ffn_down.6.weight",
"offset": 6902120448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.6.weight",
"offset": 6958743552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_gate.7.weight",
"offset": 7015366656,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_down.7.weight",
"offset": 7071989760,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.ffn_up.7.weight",
"offset": 7128612864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.4.attn_norm.weight",
"offset": 7185235968,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 7185260544,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.5.ffn_gate.0.weight",
"offset": 7185285120,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.5.ffn_down.0.weight",
"offset": 7241908224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.0.weight",
"offset": 7298531328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.1.weight",
"offset": 7355154432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.1.weight",
"offset": 7411777536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.1.weight",
"offset": 7468400640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.2.weight",
"offset": 7525023744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.2.weight",
"offset": 7581646848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.2.weight",
"offset": 7638269952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.3.weight",
"offset": 7694893056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.3.weight",
"offset": 7751516160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.3.weight",
"offset": 7808139264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.4.weight",
"offset": 7864762368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.4.weight",
"offset": 7921385472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.4.weight",
"offset": 7978008576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.5.weight",
"offset": 8034631680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.5.weight",
"offset": 8091254784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 8147877888,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.5.attn_k.weight",
"offset": 8147976192,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 8154660864,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.5.attn_q.weight",
"offset": 8175894528,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.5.attn_v.weight",
"offset": 8197128192,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.5.ffn_up.5.weight",
"offset": 8203812864,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.5.ffn_gate.6.weight",
"offset": 8260435968,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.6.weight",
"offset": 8317059072,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.6.weight",
"offset": 8373682176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_gate.7.weight",
"offset": 8430305280,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_down.7.weight",
"offset": 8486928384,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.ffn_up.7.weight",
"offset": 8543551488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.5.attn_norm.weight",
"offset": 8600174592,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 8600199168,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.6.ffn_gate.0.weight",
"offset": 8600223744,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.6.ffn_down.0.weight",
"offset": 8656846848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.0.weight",
"offset": 8713469952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.1.weight",
"offset": 8770093056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.1.weight",
"offset": 8826716160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.1.weight",
"offset": 8883339264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.2.weight",
"offset": 8939962368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.2.weight",
"offset": 8996585472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.2.weight",
"offset": 9053208576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.3.weight",
"offset": 9109831680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.3.weight",
"offset": 9166454784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.3.weight",
"offset": 9223077888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.4.weight",
"offset": 9279700992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.4.weight",
"offset": 9336324096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.4.weight",
"offset": 9392947200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.5.weight",
"offset": 9449570304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 9506193408,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.6.attn_k.weight",
"offset": 9506291712,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 9512976384,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.6.attn_q.weight",
"offset": 9534210048,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.6.attn_v.weight",
"offset": 9555443712,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.6.ffn_down.5.weight",
"offset": 9562128384,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.6.ffn_up.5.weight",
"offset": 9618751488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.6.weight",
"offset": 9675374592,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.6.weight",
"offset": 9731997696,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.6.weight",
"offset": 9788620800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_gate.7.weight",
"offset": 9845243904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_down.7.weight",
"offset": 9901867008,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.ffn_up.7.weight",
"offset": 9958490112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.6.attn_norm.weight",
"offset": 10015113216,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 10015137792,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.7.ffn_gate.0.weight",
"offset": 10015162368,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.7.ffn_down.0.weight",
"offset": 10071785472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.0.weight",
"offset": 10128408576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.1.weight",
"offset": 10185031680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.1.weight",
"offset": 10241654784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.1.weight",
"offset": 10298277888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.2.weight",
"offset": 10354900992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.2.weight",
"offset": 10411524096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.2.weight",
"offset": 10468147200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.3.weight",
"offset": 10524770304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.3.weight",
"offset": 10581393408,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.3.weight",
"offset": 10638016512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.4.weight",
"offset": 10694639616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.4.weight",
"offset": 10751262720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.4.weight",
"offset": 10807885824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 10864508928,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.7.attn_k.weight",
"offset": 10864607232,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 10871291904,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.7.attn_q.weight",
"offset": 10892525568,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.7.attn_v.weight",
"offset": 10913759232,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.7.ffn_gate.5.weight",
"offset": 10920443904,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.7.ffn_down.5.weight",
"offset": 10977067008,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.5.weight",
"offset": 11033690112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.6.weight",
"offset": 11090313216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.6.weight",
"offset": 11146936320,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.6.weight",
"offset": 11203559424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_gate.7.weight",
"offset": 11260182528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_down.7.weight",
"offset": 11316805632,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.ffn_up.7.weight",
"offset": 11373428736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.7.attn_norm.weight",
"offset": 11430051840,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 11430076416,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.8.ffn_gate.0.weight",
"offset": 11430100992,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.8.ffn_down.0.weight",
"offset": 11486724096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.0.weight",
"offset": 11543347200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.1.weight",
"offset": 11599970304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.1.weight",
"offset": 11656593408,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.1.weight",
"offset": 11713216512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.2.weight",
"offset": 11769839616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.2.weight",
"offset": 11826462720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.2.weight",
"offset": 11883085824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.3.weight",
"offset": 11939708928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.3.weight",
"offset": 11996332032,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.3.weight",
"offset": 12052955136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.4.weight",
"offset": 12109578240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.4.weight",
"offset": 12166201344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 12222824448,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.8.attn_k.weight",
"offset": 12222922752,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 12229607424,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.8.attn_q.weight",
"offset": 12250841088,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.8.attn_v.weight",
"offset": 12272074752,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.8.ffn_up.4.weight",
"offset": 12278759424,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.8.ffn_gate.5.weight",
"offset": 12335382528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.5.weight",
"offset": 12392005632,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.5.weight",
"offset": 12448628736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.6.weight",
"offset": 12505251840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.6.weight",
"offset": 12561874944,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.6.weight",
"offset": 12618498048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_gate.7.weight",
"offset": 12675121152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_down.7.weight",
"offset": 12731744256,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.ffn_up.7.weight",
"offset": 12788367360,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.8.attn_norm.weight",
"offset": 12844990464,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 12845015040,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.9.ffn_gate.0.weight",
"offset": 12845039616,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.9.ffn_down.0.weight",
"offset": 12901662720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.0.weight",
"offset": 12958285824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.1.weight",
"offset": 13014908928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.1.weight",
"offset": 13071532032,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.1.weight",
"offset": 13128155136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.2.weight",
"offset": 13184778240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.2.weight",
"offset": 13241401344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.2.weight",
"offset": 13298024448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.3.weight",
"offset": 13354647552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.3.weight",
"offset": 13411270656,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.3.weight",
"offset": 13467893760,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.4.weight",
"offset": 13524516864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 13581139968,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.9.attn_k.weight",
"offset": 13581238272,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 13587922944,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.9.attn_q.weight",
"offset": 13609156608,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.9.attn_v.weight",
"offset": 13630390272,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.10.ffn_gate.0.weight",
"offset": 13637074944,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.10.ffn_down.0.weight",
"offset": 13693698048,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.0.weight",
"offset": 13750321152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.1.weight",
"offset": 13806944256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.1.weight",
"offset": 13863567360,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.1.weight",
"offset": 13920190464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.2.weight",
"offset": 13976813568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.2.weight",
"offset": 14033436672,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.2.weight",
"offset": 14090059776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.3.weight",
"offset": 14146682880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.3.weight",
"offset": 14203305984,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.3.weight",
"offset": 14259929088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 14316552192,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.10.attn_k.weight",
"offset": 14316650496,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 14323335168,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.10.attn_q.weight",
"offset": 14344568832,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.10.attn_v.weight",
"offset": 14365802496,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.9.ffn_down.4.weight",
"offset": 14372487168,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.9.ffn_up.4.weight",
"offset": 14429110272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.5.weight",
"offset": 14485733376,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.5.weight",
"offset": 14542356480,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.5.weight",
"offset": 14598979584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.6.weight",
"offset": 14655602688,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.6.weight",
"offset": 14712225792,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.6.weight",
"offset": 14768848896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_gate.7.weight",
"offset": 14825472000,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_down.7.weight",
"offset": 14882095104,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.ffn_up.7.weight",
"offset": 14938718208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.9.attn_norm.weight",
"offset": 14995341312,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 14995365888,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.10.ffn_gate.4.weight",
"offset": 14995390464,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.10.ffn_down.4.weight",
"offset": 15052013568,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.4.weight",
"offset": 15108636672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.5.weight",
"offset": 15165259776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.5.weight",
"offset": 15221882880,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.5.weight",
"offset": 15278505984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.6.weight",
"offset": 15335129088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.6.weight",
"offset": 15391752192,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.6.weight",
"offset": 15448375296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_gate.7.weight",
"offset": 15504998400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_down.7.weight",
"offset": 15561621504,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.ffn_up.7.weight",
"offset": 15618244608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.10.attn_norm.weight",
"offset": 15674867712,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 15674892288,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.11.ffn_gate.0.weight",
"offset": 15674916864,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.11.ffn_down.0.weight",
"offset": 15731539968,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.0.weight",
"offset": 15788163072,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.1.weight",
"offset": 15844786176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.1.weight",
"offset": 15901409280,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.1.weight",
"offset": 15958032384,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.2.weight",
"offset": 16014655488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.2.weight",
"offset": 16071278592,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.2.weight",
"offset": 16127901696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.3.weight",
"offset": 16184524800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.3.weight",
"offset": 16241147904,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 16297771008,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.11.attn_k.weight",
"offset": 16297869312,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 16304553984,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.11.attn_q.weight",
"offset": 16325787648,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.11.attn_v.weight",
"offset": 16347021312,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.11.ffn_up.3.weight",
"offset": 16353705984,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.11.ffn_gate.4.weight",
"offset": 16410329088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.4.weight",
"offset": 16466952192,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.4.weight",
"offset": 16523575296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.5.weight",
"offset": 16580198400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.5.weight",
"offset": 16636821504,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.5.weight",
"offset": 16693444608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.6.weight",
"offset": 16750067712,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.6.weight",
"offset": 16806690816,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.6.weight",
"offset": 16863313920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_gate.7.weight",
"offset": 16919937024,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_down.7.weight",
"offset": 16976560128,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.ffn_up.7.weight",
"offset": 17033183232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.11.attn_norm.weight",
"offset": 17089806336,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 17089830912,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.12.ffn_gate.0.weight",
"offset": 17089855488,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.12.ffn_down.0.weight",
"offset": 17146478592,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.0.weight",
"offset": 17203101696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.1.weight",
"offset": 17259724800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.1.weight",
"offset": 17316347904,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.1.weight",
"offset": 17372971008,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.2.weight",
"offset": 17429594112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.2.weight",
"offset": 17486217216,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.2.weight",
"offset": 17542840320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.3.weight",
"offset": 17599463424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 17656086528,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.12.attn_k.weight",
"offset": 17656184832,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 17662869504,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.12.attn_q.weight",
"offset": 17684103168,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.12.attn_v.weight",
"offset": 17705336832,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.12.ffn_down.3.weight",
"offset": 17712021504,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.12.ffn_up.3.weight",
"offset": 17768644608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.4.weight",
"offset": 17825267712,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.4.weight",
"offset": 17881890816,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.4.weight",
"offset": 17938513920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.5.weight",
"offset": 17995137024,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.5.weight",
"offset": 18051760128,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.5.weight",
"offset": 18108383232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.6.weight",
"offset": 18165006336,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.6.weight",
"offset": 18221629440,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.6.weight",
"offset": 18278252544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_gate.7.weight",
"offset": 18334875648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_down.7.weight",
"offset": 18391498752,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.ffn_up.7.weight",
"offset": 18448121856,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.12.attn_norm.weight",
"offset": 18504744960,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 18504769536,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.13.ffn_gate.0.weight",
"offset": 18504794112,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.13.ffn_down.0.weight",
"offset": 18561417216,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.0.weight",
"offset": 18618040320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.1.weight",
"offset": 18674663424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.1.weight",
"offset": 18731286528,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.1.weight",
"offset": 18787909632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.2.weight",
"offset": 18844532736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.2.weight",
"offset": 18901155840,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.2.weight",
"offset": 18957778944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 19014402048,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.13.attn_k.weight",
"offset": 19014500352,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 19021185024,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.13.attn_q.weight",
"offset": 19042418688,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.13.attn_v.weight",
"offset": 19063652352,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.13.ffn_gate.3.weight",
"offset": 19070337024,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.13.ffn_down.3.weight",
"offset": 19126960128,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.3.weight",
"offset": 19183583232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.4.weight",
"offset": 19240206336,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.4.weight",
"offset": 19296829440,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.4.weight",
"offset": 19353452544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.5.weight",
"offset": 19410075648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.5.weight",
"offset": 19466698752,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.5.weight",
"offset": 19523321856,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.6.weight",
"offset": 19579944960,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.6.weight",
"offset": 19636568064,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.6.weight",
"offset": 19693191168,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_gate.7.weight",
"offset": 19749814272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_down.7.weight",
"offset": 19806437376,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.ffn_up.7.weight",
"offset": 19863060480,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.13.attn_norm.weight",
"offset": 19919683584,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 19919708160,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.14.ffn_gate.0.weight",
"offset": 19919732736,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.14.ffn_down.0.weight",
"offset": 19976355840,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.0.weight",
"offset": 20032978944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.1.weight",
"offset": 20089602048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.1.weight",
"offset": 20146225152,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.1.weight",
"offset": 20202848256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.2.weight",
"offset": 20259471360,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.2.weight",
"offset": 20316094464,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 20372717568,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.14.attn_k.weight",
"offset": 20372815872,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 20379500544,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.14.attn_q.weight",
"offset": 20400734208,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.14.attn_v.weight",
"offset": 20421967872,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.14.ffn_up.2.weight",
"offset": 20428652544,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.14.ffn_gate.3.weight",
"offset": 20485275648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.3.weight",
"offset": 20541898752,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.3.weight",
"offset": 20598521856,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.4.weight",
"offset": 20655144960,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.4.weight",
"offset": 20711768064,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.4.weight",
"offset": 20768391168,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.5.weight",
"offset": 20825014272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.5.weight",
"offset": 20881637376,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.5.weight",
"offset": 20938260480,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.6.weight",
"offset": 20994883584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.6.weight",
"offset": 21051506688,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.6.weight",
"offset": 21108129792,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_gate.7.weight",
"offset": 21164752896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_down.7.weight",
"offset": 21221376000,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.ffn_up.7.weight",
"offset": 21277999104,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.14.attn_norm.weight",
"offset": 21334622208,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 21334646784,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.15.ffn_gate.0.weight",
"offset": 21334671360,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.15.ffn_down.0.weight",
"offset": 21391294464,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.0.weight",
"offset": 21447917568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.1.weight",
"offset": 21504540672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.1.weight",
"offset": 21561163776,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.1.weight",
"offset": 21617786880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.2.weight",
"offset": 21674409984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 21731033088,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.15.attn_k.weight",
"offset": 21731131392,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 21737816064,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.15.attn_q.weight",
"offset": 21759049728,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.15.attn_v.weight",
"offset": 21780283392,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.15.ffn_down.2.weight",
"offset": 21786968064,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.15.ffn_up.2.weight",
"offset": 21843591168,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.3.weight",
"offset": 21900214272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.3.weight",
"offset": 21956837376,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.3.weight",
"offset": 22013460480,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.4.weight",
"offset": 22070083584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.4.weight",
"offset": 22126706688,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.4.weight",
"offset": 22183329792,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.5.weight",
"offset": 22239952896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.5.weight",
"offset": 22296576000,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.5.weight",
"offset": 22353199104,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.6.weight",
"offset": 22409822208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.6.weight",
"offset": 22466445312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.6.weight",
"offset": 22523068416,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_gate.7.weight",
"offset": 22579691520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_down.7.weight",
"offset": 22636314624,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.ffn_up.7.weight",
"offset": 22692937728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.15.attn_norm.weight",
"offset": 22749560832,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 22749585408,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.16.ffn_gate.0.weight",
"offset": 22749609984,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.16.ffn_down.0.weight",
"offset": 22806233088,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.0.weight",
"offset": 22862856192,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.1.weight",
"offset": 22919479296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.1.weight",
"offset": 22976102400,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.1.weight",
"offset": 23032725504,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 23089348608,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.16.attn_k.weight",
"offset": 23089446912,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 23096131584,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.16.attn_q.weight",
"offset": 23117365248,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.16.attn_v.weight",
"offset": 23138598912,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.16.ffn_gate.2.weight",
"offset": 23145283584,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.16.ffn_down.2.weight",
"offset": 23201906688,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.2.weight",
"offset": 23258529792,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.3.weight",
"offset": 23315152896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.3.weight",
"offset": 23371776000,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "\u0000\u0000\u0000\n\ufffd\u0006\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.2",
"offset": 23428399104,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.4.weight",
"offset": 23485022208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.4.weight",
"offset": 23541645312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.4.weight",
"offset": 23598268416,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.5.weight",
"offset": 23654891520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.5.weight",
"offset": 23711514624,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.5.weight",
"offset": 23768137728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.6.weight",
"offset": 23824760832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.6.weight",
"offset": 23881383936,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.6.weight",
"offset": 23938007040,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_gate.7.weight",
"offset": 23994630144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_down.7.weight",
"offset": 24051253248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.ffn_up.7.weight",
"offset": 24107876352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.16.attn_norm.weight",
"offset": 24164499456,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 24164524032,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.17.ffn_gate.0.weight",
"offset": 24164548608,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.17.ffn_down.0.weight",
"offset": 24221171712,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.0.weight",
"offset": 24277794816,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.1.weight",
"offset": 24334417920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.1.weight",
"offset": 24391041024,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 24447664128,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.17.attn_k.weight",
"offset": 24447762432,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 24454447104,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.17.attn_q.weight",
"offset": 24475680768,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.17.attn_v.weight",
"offset": 24496914432,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.17.ffn_up.1.weight",
"offset": 24503599104,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.17.ffn_gate.2.weight",
"offset": 24560222208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.2.weight",
"offset": 24616845312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.2.weight",
"offset": 24673468416,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.3.weight",
"offset": 24730091520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.3.weight",
"offset": 24786714624,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.3.weight",
"offset": 24843337728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.4.weight",
"offset": 24899960832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.4.weight",
"offset": 24956583936,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.4.weight",
"offset": 25013207040,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.5.weight",
"offset": 25069830144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.5.weight",
"offset": 25126453248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.5.weight",
"offset": 25183076352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.6.weight",
"offset": 25239699456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.6.weight",
"offset": 25296322560,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.6.weight",
"offset": 25352945664,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_gate.7.weight",
"offset": 25409568768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_down.7.weight",
"offset": 25466191872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.ffn_up.7.weight",
"offset": 25522814976,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.17.attn_norm.weight",
"offset": 25579438080,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 25579462656,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.18.ffn_gate.0.weight",
"offset": 25579487232,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.18.ffn_down.0.weight",
"offset": 25636110336,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.0.weight",
"offset": 25692733440,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.1.weight",
"offset": 25749356544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 25805979648,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.18.attn_k.weight",
"offset": 25806077952,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 25812762624,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.18.attn_q.weight",
"offset": 25833996288,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.18.attn_v.weight",
"offset": 25855229952,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.18.ffn_down.1.weight",
"offset": 25861914624,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.18.ffn_up.1.weight",
"offset": 25918537728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.2.weight",
"offset": 25975160832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.2.weight",
"offset": 26031783936,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.2.weight",
"offset": 26088407040,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.3.weight",
"offset": 26145030144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.3.weight",
"offset": 26201653248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.3.weight",
"offset": 26258276352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.4.weight",
"offset": 26314899456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.4.weight",
"offset": 26371522560,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.4.weight",
"offset": 26428145664,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.5.weight",
"offset": 26484768768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.5.weight",
"offset": 26541391872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.5.weight",
"offset": 26598014976,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.6.weight",
"offset": 26654638080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.6.weight",
"offset": 26711261184,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.6.weight",
"offset": 26767884288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_gate.7.weight",
"offset": 26824507392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_down.7.weight",
"offset": 26881130496,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.ffn_up.7.weight",
"offset": 26937753600,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.18.attn_norm.weight",
"offset": 26994376704,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 26994401280,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.19.ffn_gate.0.weight",
"offset": 26994425856,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.19.ffn_down.0.weight",
"offset": 27051048960,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.0.weight",
"offset": 27107672064,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 27164295168,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.19.attn_k.weight",
"offset": 27164393472,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 27171078144,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.19.attn_q.weight",
"offset": 27192311808,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.19.attn_v.weight",
"offset": 27213545472,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.19.ffn_gate.1.weight",
"offset": 27220230144,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.19.ffn_down.1.weight",
"offset": 27276853248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.1.weight",
"offset": 27333476352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.2.weight",
"offset": 27390099456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.2.weight",
"offset": 27446722560,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.2.weight",
"offset": 27503345664,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.3.weight",
"offset": 27559968768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.3.weight",
"offset": 27616591872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.3.weight",
"offset": 27673214976,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.4.weight",
"offset": 27729838080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.4.weight",
"offset": 27786461184,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.4.weight",
"offset": 27843084288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.5.weight",
"offset": 27899707392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.5.weight",
"offset": 27956330496,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.5.weight",
"offset": 28012953600,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.6.weight",
"offset": 28069576704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.6.weight",
"offset": 28126199808,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.6.weight",
"offset": 28182822912,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_gate.7.weight",
"offset": 28239446016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_down.7.weight",
"offset": 28296069120,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.ffn_up.7.weight",
"offset": 28352692224,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.19.attn_norm.weight",
"offset": 28409315328,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 28409339904,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.20.ffn_gate.0.weight",
"offset": 28409364480,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.20.ffn_down.0.weight",
"offset": 28465987584,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 28522610688,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.20.attn_k.weight",
"offset": 28522708992,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 28529393664,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.20.attn_q.weight",
"offset": 28550627328,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.20.attn_v.weight",
"offset": 28571860992,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.20.ffn_up.0.weight",
"offset": 28578545664,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.20.ffn_gate.1.weight",
"offset": 28635168768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.1.weight",
"offset": 28691791872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.1.weight",
"offset": 28748414976,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.2.weight",
"offset": 28805038080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.2.weight",
"offset": 28861661184,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.2.weight",
"offset": 28918284288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.3.weight",
"offset": 28974907392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.3.weight",
"offset": 29031530496,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.3.weight",
"offset": 29088153600,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.4.weight",
"offset": 29144776704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.4.weight",
"offset": 29201399808,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.4.weight",
"offset": 29258022912,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.5.weight",
"offset": 29314646016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.5.weight",
"offset": 29371269120,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.5.weight",
"offset": 29427892224,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.6.weight",
"offset": 29484515328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.6.weight",
"offset": 29541138432,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.6.weight",
"offset": 29597761536,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_gate.7.weight",
"offset": 29654384640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_down.7.weight",
"offset": 29711007744,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.ffn_up.7.weight",
"offset": 29767630848,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.20.attn_norm.weight",
"offset": 29824253952,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 29824278528,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.21.ffn_gate.0.weight",
"offset": 29824303104,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.21.ffn_gate_inp.weight",
"offset": 29880926208,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.21.attn_k.weight",
"offset": 29881024512,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 29887709184,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.21.attn_q.weight",
"offset": 29908942848,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.21.attn_v.weight",
"offset": 29930176512,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.21.ffn_down.0.weight",
"offset": 29936861184,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.21.ffn_up.0.weight",
"offset": 29993484288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.1.weight",
"offset": 30050107392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.1.weight",
"offset": 30106730496,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.1.weight",
"offset": 30163353600,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.2.weight",
"offset": 30219976704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.2.weight",
"offset": 30276599808,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.2.weight",
"offset": 30333222912,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.3.weight",
"offset": 30389846016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.3.weight",
"offset": 30446469120,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.3.weight",
"offset": 30503092224,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.4.weight",
"offset": 30559715328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.4.weight",
"offset": 30616338432,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.4.weight",
"offset": 30672961536,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.5.weight",
"offset": 30729584640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.5.weight",
"offset": 30786207744,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.5.weight",
"offset": 30842830848,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.6.weight",
"offset": 30899453952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.6.weight",
"offset": 30956077056,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.6.weight",
"offset": 31012700160,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_gate.7.weight",
"offset": 31069323264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_down.7.weight",
"offset": 31125946368,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.ffn_up.7.weight",
"offset": 31182569472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.21.attn_norm.weight",
"offset": 31239192576,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 31239217152,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 31239241728,
"shape": [
6144,
8
],
"size": 24576,
"type": 1
},
{
"name": "blk.22.attn_k.weight",
"offset": 31239340032,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 31246024704,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.22.attn_q.weight",
"offset": 31267258368,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.22.attn_v.weight",
"offset": 31288492032,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.22.ffn_gate.0.weight",
"offset": 31295176704,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.22.ffn_down.0.weight",
"offset": 31351799808,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.0.weight",
"offset": 31408422912,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.1.weight",
"offset": 31465046016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.1.weight",
"offset": 31521669120,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.1.weight",
"offset": 31578292224,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.2.weight",
"offset": 31634915328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.2.weight",
"offset": 31691538432,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.2.weight",
"offset": 31748161536,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.3.weight",
"offset": 31804784640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.3.weight",
"offset": 31861407744,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.3.weight",
"offset": 31918030848,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.4.weight",
"offset": 31974653952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.4.weight",
"offset": 32031277056,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.4.weight",
"offset": 32087900160,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.5.weight",
"offset": 32144523264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.5.weight",
"offset": 32201146368,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.5.weight",
"offset": 32257769472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.6.weight",
"offset": 32314392576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.6.weight",
"offset": 32371015680,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.6.weight",
"offset": 32427638784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_gate.7.weight",
"offset": 32484261888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_down.7.weight",
"offset": 32540884992,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.ffn_up.7.weight",
"offset": 32597508096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.22.attn_norm.weight",
"offset": 32654131200,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 32654155776,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 32654180352,
"shape": [
6144,
1024
],
"size": 24576,
"type": 8
},
{
"name": "blk.23.attn_q.weight",
"offset": 32660865024,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.23.attn_v.weight",
"offset": 32682098688,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.23.ffn_gate.0.weight",
"offset": 32688783360,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.23.ffn_down.0.weight",
"offset": 32745406464,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.0.weight",
"offset": 32802029568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.1.weight",
"offset": 32858652672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.1.weight",
"offset": 32915275776,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.1.weight",
"offset": 32971898880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.2.weight",
"offset": 33028521984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.2.weight",
"offset": 33085145088,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.2.weight",
"offset": 33141768192,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.3.weight",
"offset": 33198391296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.3.weight",
"offset": 33255014400,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.3.weight",
"offset": 33311637504,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.4.weight",
"offset": 33368260608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.4.weight",
"offset": 33424883712,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.4.weight",
"offset": 33481506816,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.5.weight",
"offset": 33538129920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.5.weight",
"offset": 33594753024,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.5.weight",
"offset": 33651376128,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.6.weight",
"offset": 33707999232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.6.weight",
"offset": 33764622336,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.6.weight",
"offset": 33821245440,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate.7.weight",
"offset": 33877868544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_down.7.weight",
"offset": 33934491648,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_up.7.weight",
"offset": 33991114752,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 34047737856,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 34047836160,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 34047860736,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.23.attn_output.weight",
"offset": 34047885312,
"shape": [
6144,
6144
],
"size": 24576,
"type": 2
},
{
"name": "blk.24.attn_k.weight",
"offset": 34069118976,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.24.attn_q.weight",
"offset": 34075803648,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.24.ffn_gate.0.weight",
"offset": 34097037312,
"shape": [
6144,
16384
],
"size": 21233664,
"type": 2
},
{
"name": "blk.24.ffn_down.0.weight",
"offset": 34153660416,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.0.weight",
"offset": 34210283520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.1.weight",
"offset": 34266906624,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.1.weight",
"offset": 34323529728,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.1.weight",
"offset": 34380152832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.2.weight",
"offset": 34436775936,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.2.weight",
"offset": 34493399040,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.2.weight",
"offset": 34550022144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.3.weight",
"offset": 34606645248,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.3.weight",
"offset": 34663268352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.3.weight",
"offset": 34719891456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.4.weight",
"offset": 34776514560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.4.weight",
"offset": 34833137664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.4.weight",
"offset": 34889760768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.5.weight",
"offset": 34946383872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.5.weight",
"offset": 35003006976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.5.weight",
"offset": 35059630080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.6.weight",
"offset": 35116253184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.6.weight",
"offset": 35172876288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.6.weight",
"offset": 35229499392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate.7.weight",
"offset": 35286122496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_down.7.weight",
"offset": 35342745600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_up.7.weight",
"offset": 35399368704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 35455991808,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 35456090112,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 35456114688,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.24.attn_output.weight",
"offset": 35456139264,
"shape": [
6144,
6144
],
"size": 24576,
"type": 2
},
{
"name": "\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.29.ff",
"offset": 35477372928,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.25.attn_q.weight",
"offset": 35484057600,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.25.ffn_gate.0.weight",
"offset": 35505291264,
"shape": [
6144,
16384
],
"size": 21233664,
"type": 2
},
{
"name": "blk.25.ffn_down.0.weight",
"offset": 35561914368,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.0.weight",
"offset": 35618537472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.1.weight",
"offset": 35675160576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.1.weight",
"offset": 35731783680,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.1.weight",
"offset": 35788406784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.2.weight",
"offset": 35845029888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.2.weight",
"offset": 35901652992,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.2.weight",
"offset": 35958276096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.3.weight",
"offset": 36014899200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.3.weight",
"offset": 36071522304,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.3.weight",
"offset": 36128145408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.4.weight",
"offset": 36184768512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.4.weight",
"offset": 36241391616,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.4.weight",
"offset": 36298014720,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.5.weight",
"offset": 36354637824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.5.weight",
"offset": 36411260928,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.5.weight",
"offset": 36467884032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.6.weight",
"offset": 36524507136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.6.weight",
"offset": 36581130240,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.6.weight",
"offset": 36637753344,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate.7.weight",
"offset": 36694376448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_down.7.weight",
"offset": 36750999552,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_up.7.weight",
"offset": 36807622656,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 36864245760,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 36864344064,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 36864368640,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 36864393216,
"shape": [
6144,
1024
],
"size": 24576,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 36871077888,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.25.attn_v.weight",
"offset": 36892311552,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.26.ffn_gate.0.weight",
"offset": 36898996224,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.26.ffn_down.0.weight",
"offset": 36955619328,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.0.weight",
"offset": 37012242432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.1.weight",
"offset": 37068865536,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.1.weight",
"offset": 37125488640,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.1.weight",
"offset": 37182111744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.2.weight",
"offset": 37238734848,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.2.weight",
"offset": 37295357952,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.2.weight",
"offset": 37351981056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.3.weight",
"offset": 37408604160,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.3.weight",
"offset": 37465227264,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.3.weight",
"offset": 37521850368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.4.weight",
"offset": 37578473472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.4.weight",
"offset": 37635096576,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.4.weight",
"offset": 37691719680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.5.weight",
"offset": 37748342784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.5.weight",
"offset": 37804965888,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.5.weight",
"offset": 37861588992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.6.weight",
"offset": 37918212096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.6.weight",
"offset": 37974835200,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_up.6.weight",
"offset": 38031458304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate.7.weight",
"offset": 38088081408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_down.7.weight",
"offset": 38144704512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 38201327616,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.26.attn_k.weight",
"offset": 38201425920,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 38208110592,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.26.attn_q.weight",
"offset": 38229344256,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.26.attn_v.weight",
"offset": 38250577920,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.26.ffn_up.7.weight",
"offset": 38257262592,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.26.attn_norm.weight",
"offset": 38313885696,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 38313910272,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.27.ffn_gate.0.weight",
"offset": 38313934848,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.27.ffn_down.0.weight",
"offset": 38370557952,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.0.weight",
"offset": 38427181056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.1.weight",
"offset": 38483804160,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.1.weight",
"offset": 38540427264,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.1.weight",
"offset": 38597050368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.2.weight",
"offset": 38653673472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.2.weight",
"offset": 38710296576,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.2.weight",
"offset": 38766919680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.3.weight",
"offset": 38823542784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.3.weight",
"offset": 38880165888,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.3.weight",
"offset": 38936788992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.4.weight",
"offset": 38993412096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.4.weight",
"offset": 39050035200,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.4.weight",
"offset": 39106658304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.5.weight",
"offset": 39163281408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.5.weight",
"offset": 39219904512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.5.weight",
"offset": 39276527616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.6.weight",
"offset": 39333150720,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_down.6.weight",
"offset": 39389773824,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_up.6.weight",
"offset": 39446396928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate.7.weight",
"offset": 39503020032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 39559643136,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.27.attn_k.weight",
"offset": 39559741440,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 39566426112,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.27.attn_q.weight",
"offset": 39587659776,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.27.attn_v.weight",
"offset": 39608893440,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.27.ffn_down.7.weight",
"offset": 39615578112,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.27.ffn_up.7.weight",
"offset": 39672201216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.27.attn_norm.weight",
"offset": 39728824320,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 39728848896,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.28.ffn_gate.0.weight",
"offset": 39728873472,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.28.ffn_down.0.weight",
"offset": 39785496576,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.0.weight",
"offset": 39842119680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.1.weight",
"offset": 39898742784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.1.weight",
"offset": 39955365888,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.1.weight",
"offset": 40011988992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.2.weight",
"offset": 40068612096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.2.weight",
"offset": 40125235200,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.2.weight",
"offset": 40181858304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.3.weight",
"offset": 40238481408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.3.weight",
"offset": 40295104512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.3.weight",
"offset": 40351727616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.4.weight",
"offset": 40408350720,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.4.weight",
"offset": 40464973824,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.4.weight",
"offset": 40521596928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.5.weight",
"offset": 40578220032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.5.weight",
"offset": 40634843136,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.5.weight",
"offset": 40691466240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate.6.weight",
"offset": 40748089344,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_down.6.weight",
"offset": 40804712448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.6.weight",
"offset": 40861335552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_gate_inp.weight",
"offset": 40917958656,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.28.attn_k.weight",
"offset": 40918056960,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 40924741632,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.28.attn_q.weight",
"offset": 40945975296,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.28.attn_v.weight",
"offset": 40967208960,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.28.ffn_gate.7.weight",
"offset": 40973893632,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.28.ffn_down.7.weight",
"offset": 41030516736,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.ffn_up.7.weight",
"offset": 41087139840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.28.attn_norm.weight",
"offset": 41143762944,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 41143787520,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.29.ffn_gate.0.weight",
"offset": 41143812096,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.29.ffn_down.0.weight",
"offset": 41200435200,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.0.weight",
"offset": 41257058304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.1.weight",
"offset": 41313681408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.1.weight",
"offset": 41370304512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.1.weight",
"offset": 41426927616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.2.weight",
"offset": 41483550720,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.2.weight",
"offset": 41540173824,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.2.weight",
"offset": 41596796928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.3.weight",
"offset": 41653420032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.3.weight",
"offset": 41710043136,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.3.weight",
"offset": 41766666240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.4.weight",
"offset": 41823289344,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.4.weight",
"offset": 41879912448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.4.weight",
"offset": 41936535552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.5.weight",
"offset": 41993158656,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.5.weight",
"offset": 42049781760,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.5.weight",
"offset": 42106404864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate.6.weight",
"offset": 42163027968,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.6.weight",
"offset": 42219651072,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_gate_inp.weight",
"offset": 42276274176,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.29.attn_k.weight",
"offset": 42276372480,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 42283057152,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.29.attn_q.weight",
"offset": 42304290816,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.29.attn_v.weight",
"offset": 42325524480,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.29.ffn_up.6.weight",
"offset": 42332209152,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.29.ffn_gate.7.weight",
"offset": 42388832256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_down.7.weight",
"offset": 42445455360,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.ffn_up.7.weight",
"offset": 42502078464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.29.attn_norm.weight",
"offset": 42558701568,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 42558726144,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.30.ffn_gate.0.weight",
"offset": 42558750720,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.30.ffn_down.0.weight",
"offset": 42615373824,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.0.weight",
"offset": 42671996928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.1.weight",
"offset": 42728620032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.1.weight",
"offset": 42785243136,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.1.weight",
"offset": 42841866240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.2.weight",
"offset": 42898489344,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.2.weight",
"offset": 42955112448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.2.weight",
"offset": 43011735552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.3.weight",
"offset": 43068358656,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.3.weight",
"offset": 43124981760,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.3.weight",
"offset": 43181604864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.4.weight",
"offset": 43238227968,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.4.weight",
"offset": 43294851072,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.4.weight",
"offset": 43351474176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.5.weight",
"offset": 43408097280,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.5.weight",
"offset": 43464720384,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.5.weight",
"offset": 43521343488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.6.weight",
"offset": 43577966592,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate_inp.weight",
"offset": 43634589696,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.30.attn_k.weight",
"offset": 43634688000,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 43641372672,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.30.attn_q.weight",
"offset": 43662606336,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.30.attn_v.weight",
"offset": 43683840000,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.30.ffn_down.6.weight",
"offset": 43690524672,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.30.ffn_up.6.weight",
"offset": 43747147776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_gate.7.weight",
"offset": 43803770880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_down.7.weight",
"offset": 43860393984,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.ffn_up.7.weight",
"offset": 43917017088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.30.attn_norm.weight",
"offset": 43973640192,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 43973664768,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.31.ffn_gate.0.weight",
"offset": 43973689344,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.31.ffn_down.0.weight",
"offset": 44030312448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.0.weight",
"offset": 44086935552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.1.weight",
"offset": 44143558656,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.1.weight",
"offset": 44200181760,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.1.weight",
"offset": 44256804864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.2.weight",
"offset": 44313427968,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.2.weight",
"offset": 44370051072,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.2.weight",
"offset": 44426674176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.3.weight",
"offset": 44483297280,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.3.weight",
"offset": 44539920384,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.3.weight",
"offset": 44596543488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.4.weight",
"offset": 44653166592,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.4.weight",
"offset": 44709789696,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.4.weight",
"offset": 44766412800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.5.weight",
"offset": 44823035904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.5.weight",
"offset": 44879659008,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.5.weight",
"offset": 44936282112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate_inp.weight",
"offset": 44992905216,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.31.attn_k.weight",
"offset": 44993003520,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 44999688192,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.31.attn_q.weight",
"offset": 45020921856,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.31.attn_v.weight",
"offset": 45042155520,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.31.ffn_gate.6.weight",
"offset": 45048840192,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.31.ffn_down.6.weight",
"offset": 45105463296,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.6.weight",
"offset": 45162086400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_gate.7.weight",
"offset": 45218709504,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_down.7.weight",
"offset": 45275332608,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.ffn_up.7.weight",
"offset": 45331955712,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.31.attn_norm.weight",
"offset": 45388578816,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 45388603392,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.32.ffn_gate.0.weight",
"offset": 45388627968,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.32.ffn_down.0.weight",
"offset": 45445251072,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.0.weight",
"offset": 45501874176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.1.weight",
"offset": 45558497280,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.1.weight",
"offset": 45615120384,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.1.weight",
"offset": 45671743488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.2.weight",
"offset": 45728366592,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.2.weight",
"offset": 45784989696,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.2.weight",
"offset": 45841612800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.3.weight",
"offset": 45898235904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.3.weight",
"offset": 45954859008,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.3.weight",
"offset": 46011482112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.4.weight",
"offset": 46068105216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.4.weight",
"offset": 46124728320,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.4.weight",
"offset": 46181351424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.5.weight",
"offset": 46237974528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.5.weight",
"offset": 46294597632,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate_inp.weight",
"offset": 46351220736,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.32.attn_k.weight",
"offset": 46351319040,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.32.attn_output.weight",
"offset": 46358003712,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.32.attn_q.weight",
"offset": 46379237376,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.32.attn_v.weight",
"offset": 46400471040,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.32.ffn_up.5.weight",
"offset": 46407155712,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.32.ffn_gate.6.weight",
"offset": 46463778816,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.6.weight",
"offset": 46520401920,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.6.weight",
"offset": 46577025024,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_gate.7.weight",
"offset": 46633648128,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_down.7.weight",
"offset": 46690271232,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.ffn_up.7.weight",
"offset": 46746894336,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.32.attn_norm.weight",
"offset": 46803517440,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 46803542016,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.33.ffn_gate.0.weight",
"offset": 46803566592,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.33.ffn_down.0.weight",
"offset": 46860189696,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.0.weight",
"offset": 46916812800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.1.weight",
"offset": 46973435904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_down.1.weight",
"offset": 47030059008,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.1.weight",
"offset": 47086682112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.2.weight",
"offset": 47143305216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_down.2.weight",
"offset": 47199928320,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.2.weight",
"offset": 47256551424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.3.weight",
"offset": 47313174528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_down.3.weight",
"offset": 47369797632,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.3.weight",
"offset": 47426420736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.4.weight",
"offset": 47483043840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "\u0000\u0000\u0000\u0000\u0000blk.37.ffn_gate.5.w",
"offset": 47539666944,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.4.weight",
"offset": 47596290048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.5.weight",
"offset": 47652913152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate_inp.weight",
"offset": 47709536256,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.33.attn_k.weight",
"offset": 47709634560,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.33.attn_output.weight",
"offset": 47716319232,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.33.attn_q.weight",
"offset": 47737552896,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.33.attn_v.weight",
"offset": 47758786560,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.33.ffn_down.5.weight",
"offset": 47765471232,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.33.ffn_up.5.weight",
"offset": 47822094336,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.6.weight",
"offset": 47878717440,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_down.6.weight",
"offset": 47935340544,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.6.weight",
"offset": 47991963648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_gate.7.weight",
"offset": 48048586752,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_down.7.weight",
"offset": 48105209856,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.ffn_up.7.weight",
"offset": 48161832960,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.33.attn_norm.weight",
"offset": 48218456064,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 48218480640,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.34.ffn_gate.0.weight",
"offset": 48218505216,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.34.ffn_down.0.weight",
"offset": 48275128320,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.0.weight",
"offset": 48331751424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.1.weight",
"offset": 48388374528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.1.weight",
"offset": 48444997632,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.1.weight",
"offset": 48501620736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.2.weight",
"offset": 48558243840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.2.weight",
"offset": 48614866944,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.2.weight",
"offset": 48671490048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.3.weight",
"offset": 48728113152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.3.weight",
"offset": 48784736256,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.3.weight",
"offset": 48841359360,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.4.weight",
"offset": 48897982464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.4.weight",
"offset": 48954605568,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.4.weight",
"offset": 49011228672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate_inp.weight",
"offset": 49067851776,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.34.attn_k.weight",
"offset": 49067950080,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.34.attn_output.weight",
"offset": 49074634752,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.34.attn_q.weight",
"offset": 49095868416,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.34.attn_v.weight",
"offset": 49117102080,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.34.ffn_gate.5.weight",
"offset": 49123786752,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.34.ffn_down.5.weight",
"offset": 49180409856,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.5.weight",
"offset": 49237032960,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.6.weight",
"offset": 49293656064,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.6.weight",
"offset": 49350279168,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.6.weight",
"offset": 49406902272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_gate.7.weight",
"offset": 49463525376,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_down.7.weight",
"offset": 49520148480,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.ffn_up.7.weight",
"offset": 49576771584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.34.attn_norm.weight",
"offset": 49633394688,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 49633419264,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.35.ffn_gate.0.weight",
"offset": 49633443840,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.35.ffn_down.0.weight",
"offset": 49690066944,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.0.weight",
"offset": 49746690048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.1.weight",
"offset": 49803313152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.1.weight",
"offset": 49859936256,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.1.weight",
"offset": 49916559360,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.2.weight",
"offset": 49973182464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.2.weight",
"offset": 50029805568,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.2.weight",
"offset": 50086428672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.3.weight",
"offset": 50143051776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.3.weight",
"offset": 50199674880,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.3.weight",
"offset": 50256297984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.4.weight",
"offset": 50312921088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.4.weight",
"offset": 50369544192,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate_inp.weight",
"offset": 50426167296,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.35.attn_k.weight",
"offset": 50426265600,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.35.attn_output.weight",
"offset": 50432950272,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.35.attn_q.weight",
"offset": 50454183936,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.35.attn_v.weight",
"offset": 50475417600,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.35.ffn_up.4.weight",
"offset": 50482102272,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.35.ffn_gate.5.weight",
"offset": 50538725376,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.5.weight",
"offset": 50595348480,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.5.weight",
"offset": 50651971584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.6.weight",
"offset": 50708594688,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.6.weight",
"offset": 50765217792,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.6.weight",
"offset": 50821840896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_gate.7.weight",
"offset": 50878464000,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_down.7.weight",
"offset": 50935087104,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.ffn_up.7.weight",
"offset": 50991710208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.35.attn_norm.weight",
"offset": 51048333312,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 51048357888,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.36.ffn_gate.0.weight",
"offset": 51048382464,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.36.ffn_down.0.weight",
"offset": 51105005568,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.0.weight",
"offset": 51161628672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.1.weight",
"offset": 51218251776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.1.weight",
"offset": 51274874880,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.1.weight",
"offset": 51331497984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.2.weight",
"offset": 51388121088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.2.weight",
"offset": 51444744192,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.2.weight",
"offset": 51501367296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.3.weight",
"offset": 51557990400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.3.weight",
"offset": 51614613504,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.3.weight",
"offset": 51671236608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.4.weight",
"offset": 51727859712,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate_inp.weight",
"offset": 51784482816,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.36.attn_k.weight",
"offset": 51784581120,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.36.attn_output.weight",
"offset": 51791265792,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.36.attn_q.weight",
"offset": 51812499456,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.36.attn_v.weight",
"offset": 51833733120,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.36.ffn_down.4.weight",
"offset": 51840417792,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.36.ffn_up.4.weight",
"offset": 51897040896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.5.weight",
"offset": 51953664000,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.5.weight",
"offset": 52010287104,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.5.weight",
"offset": 52066910208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.6.weight",
"offset": 52123533312,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.6.weight",
"offset": 52180156416,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.6.weight",
"offset": 52236779520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_gate.7.weight",
"offset": 52293402624,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_down.7.weight",
"offset": 52350025728,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.ffn_up.7.weight",
"offset": 52406648832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.36.attn_norm.weight",
"offset": 52463271936,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 52463296512,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.37.ffn_gate.0.weight",
"offset": 52463321088,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.37.ffn_down.0.weight",
"offset": 52519944192,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.0.weight",
"offset": 52576567296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate.1.weight",
"offset": 52633190400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.1.weight",
"offset": 52689813504,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.1.weight",
"offset": 52746436608,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate.2.weight",
"offset": 52803059712,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.2.weight",
"offset": 52859682816,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.2.weight",
"offset": 52916305920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate.3.weight",
"offset": 52972929024,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.3.weight",
"offset": 53029552128,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.3.weight",
"offset": 53086175232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate_inp.weight",
"offset": 53142798336,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.37.attn_k.weight",
"offset": 53142896640,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.37.attn_output.weight",
"offset": 53149581312,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.37.attn_q.weight",
"offset": 53170814976,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.37.attn_v.weight",
"offset": 53192048640,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.37.ffn_gate.4.weight",
"offset": 53198733312,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.37.ffn_down.4.weight",
"offset": 53255356416,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.4.weight",
"offset": 53311979520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "@\ufffd\ufffd\r\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.41.ff",
"offset": 53368602624,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.5.weight",
"offset": 53425225728,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.5.weight",
"offset": 53481848832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate.6.weight",
"offset": 53538471936,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.6.weight",
"offset": 53595095040,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.6.weight",
"offset": 53651718144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_gate.7.weight",
"offset": 53708341248,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_down.7.weight",
"offset": 53764964352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.ffn_up.7.weight",
"offset": 53821587456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.37.attn_norm.weight",
"offset": 53878210560,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 53878235136,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.38.ffn_gate.0.weight",
"offset": 53878259712,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.38.ffn_down.0.weight",
"offset": 53934882816,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.0.weight",
"offset": 53991505920,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.1.weight",
"offset": 54048129024,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.1.weight",
"offset": 54104752128,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.1.weight",
"offset": 54161375232,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.2.weight",
"offset": 54217998336,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.2.weight",
"offset": 54274621440,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.2.weight",
"offset": 54331244544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.3.weight",
"offset": 54387867648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.3.weight",
"offset": 54444490752,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate_inp.weight",
"offset": 54501113856,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.38.attn_k.weight",
"offset": 54501212160,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.38.attn_output.weight",
"offset": 54507896832,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.38.attn_q.weight",
"offset": 54529130496,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.38.attn_v.weight",
"offset": 54550364160,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.38.ffn_up.3.weight",
"offset": 54557048832,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.38.ffn_gate.4.weight",
"offset": 54613671936,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.4.weight",
"offset": 54670295040,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.4.weight",
"offset": 54726918144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.5.weight",
"offset": 54783541248,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.5.weight",
"offset": 54840164352,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.5.weight",
"offset": 54896787456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.6.weight",
"offset": 54953410560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.6.weight",
"offset": 55010033664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.6.weight",
"offset": 55066656768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_gate.7.weight",
"offset": 55123279872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_down.7.weight",
"offset": 55179902976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.ffn_up.7.weight",
"offset": 55236526080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.38.attn_norm.weight",
"offset": 55293149184,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 55293173760,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.39.ffn_gate.0.weight",
"offset": 55293198336,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.39.ffn_down.0.weight",
"offset": 55349821440,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.0.weight",
"offset": 55406444544,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.1.weight",
"offset": 55463067648,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.1.weight",
"offset": 55519690752,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.1.weight",
"offset": 55576313856,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.2.weight",
"offset": 55632936960,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.2.weight",
"offset": 55689560064,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.2.weight",
"offset": 55746183168,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.3.weight",
"offset": 55802806272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate_inp.weight",
"offset": 55859429376,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.39.attn_k.weight",
"offset": 55859527680,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.39.attn_output.weight",
"offset": 55866212352,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.39.attn_q.weight",
"offset": 55887446016,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.39.attn_v.weight",
"offset": 55908679680,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.39.ffn_down.3.weight",
"offset": 55915364352,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.39.ffn_up.3.weight",
"offset": 55971987456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.4.weight",
"offset": 56028610560,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.4.weight",
"offset": 56085233664,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.4.weight",
"offset": 56141856768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.5.weight",
"offset": 56198479872,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.5.weight",
"offset": 56255102976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.5.weight",
"offset": 56311726080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.6.weight",
"offset": 56368349184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.6.weight",
"offset": 56424972288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.6.weight",
"offset": 56481595392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_gate.7.weight",
"offset": 56538218496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_down.7.weight",
"offset": 56594841600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.ffn_up.7.weight",
"offset": 56651464704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.39.attn_norm.weight",
"offset": 56708087808,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 56708112384,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.40.ffn_gate.0.weight",
"offset": 56708136960,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.40.ffn_down.0.weight",
"offset": 56764760064,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.0.weight",
"offset": 56821383168,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.1.weight",
"offset": 56878006272,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.1.weight",
"offset": 56934629376,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.1.weight",
"offset": 56991252480,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.2.weight",
"offset": 57047875584,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.2.weight",
"offset": 57104498688,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.2.weight",
"offset": 57161121792,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate_inp.weight",
"offset": 57217744896,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.40.attn_k.weight",
"offset": 57217843200,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.40.attn_output.weight",
"offset": 57224527872,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.40.attn_q.weight",
"offset": 57245761536,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.40.attn_v.weight",
"offset": 57266995200,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.40.ffn_gate.3.weight",
"offset": 57273679872,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.40.ffn_down.3.weight",
"offset": 57330302976,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.3.weight",
"offset": 57386926080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.4.weight",
"offset": 57443549184,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.4.weight",
"offset": 57500172288,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.4.weight",
"offset": 57556795392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.5.weight",
"offset": 57613418496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.5.weight",
"offset": 57670041600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.5.weight",
"offset": 57726664704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.6.weight",
"offset": 57783287808,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.6.weight",
"offset": 57839910912,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.6.weight",
"offset": 57896534016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_gate.7.weight",
"offset": 57953157120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_down.7.weight",
"offset": 58009780224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.ffn_up.7.weight",
"offset": 58066403328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.40.attn_norm.weight",
"offset": 58123026432,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 58123051008,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.41.ffn_gate.0.weight",
"offset": 58123075584,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.41.ffn_down.0.weight",
"offset": 58179698688,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.0.weight",
"offset": 58236321792,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.1.weight",
"offset": 58292944896,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.1.weight",
"offset": 58349568000,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.1.weight",
"offset": 58406191104,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.2.weight",
"offset": 58462814208,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.2.weight",
"offset": 58519437312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate_inp.weight",
"offset": 58576060416,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.41.attn_k.weight",
"offset": 58576158720,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.41.attn_output.weight",
"offset": 58582843392,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.41.attn_q.weight",
"offset": 58604077056,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.41.attn_v.weight",
"offset": 58625310720,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.41.ffn_up.2.weight",
"offset": 58631995392,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.41.ffn_gate.3.weight",
"offset": 58688618496,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.3.weight",
"offset": 58745241600,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.3.weight",
"offset": 58801864704,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.4.weight",
"offset": 58858487808,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.4.weight",
"offset": 58915110912,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.4.weight",
"offset": 58971734016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.5.weight",
"offset": 59028357120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.5.weight",
"offset": 59084980224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.5.weight",
"offset": 59141603328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.6.weight",
"offset": 59198226432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_down.6.weight",
"offset": 59254849536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.6.weight",
"offset": 59311472640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_gate.7.weight",
"offset": 59368095744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "ffn_gate_inp.weight\u0002\u0000\u0000\u0000\u0000",
"offset": 59424718848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.ffn_up.7.weight",
"offset": 59481341952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.41.attn_norm.weight",
"offset": 59537965056,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 59537989632,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.42.ffn_gate.0.weight",
"offset": 59538014208,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.42.ffn_down.0.weight",
"offset": 59594637312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.0.weight",
"offset": 59651260416,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.1.weight",
"offset": 59707883520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.1.weight",
"offset": 59764506624,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.1.weight",
"offset": 59821129728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.2.weight",
"offset": 59877752832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate_inp.weight",
"offset": 59934375936,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.42.attn_k.weight",
"offset": 59934474240,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.42.attn_output.weight",
"offset": 59941158912,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.42.attn_q.weight",
"offset": 59962392576,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.42.attn_v.weight",
"offset": 59983626240,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.42.ffn_down.2.weight",
"offset": 59990310912,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.42.ffn_up.2.weight",
"offset": 60046934016,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.3.weight",
"offset": 60103557120,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.3.weight",
"offset": 60160180224,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.3.weight",
"offset": 60216803328,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.4.weight",
"offset": 60273426432,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.4.weight",
"offset": 60330049536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.4.weight",
"offset": 60386672640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.5.weight",
"offset": 60443295744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.5.weight",
"offset": 60499918848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.5.weight",
"offset": 60556541952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.6.weight",
"offset": 60613165056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.6.weight",
"offset": 60669788160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.6.weight",
"offset": 60726411264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_gate.7.weight",
"offset": 60783034368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_down.7.weight",
"offset": 60839657472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.ffn_up.7.weight",
"offset": 60896280576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.42.attn_norm.weight",
"offset": 60952903680,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 60952928256,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.43.ffn_gate.0.weight",
"offset": 60952952832,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.43.ffn_down.0.weight",
"offset": 61009575936,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.0.weight",
"offset": 61066199040,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.1.weight",
"offset": 61122822144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.1.weight",
"offset": 61179445248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.1.weight",
"offset": 61236068352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate_inp.weight",
"offset": 61292691456,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.43.attn_k.weight",
"offset": 61292789760,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.43.attn_output.weight",
"offset": 61299474432,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.43.attn_q.weight",
"offset": 61320708096,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.43.attn_v.weight",
"offset": 61341941760,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.43.ffn_gate.2.weight",
"offset": 61348626432,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.43.ffn_down.2.weight",
"offset": 61405249536,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.2.weight",
"offset": 61461872640,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.3.weight",
"offset": 61518495744,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.3.weight",
"offset": 61575118848,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.3.weight",
"offset": 61631741952,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.4.weight",
"offset": 61688365056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.4.weight",
"offset": 61744988160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.4.weight",
"offset": 61801611264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.5.weight",
"offset": 61858234368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.5.weight",
"offset": 61914857472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.5.weight",
"offset": 61971480576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.6.weight",
"offset": 62028103680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.6.weight",
"offset": 62084726784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.6.weight",
"offset": 62141349888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_gate.7.weight",
"offset": 62197972992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_down.7.weight",
"offset": 62254596096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.ffn_up.7.weight",
"offset": 62311219200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.43.attn_norm.weight",
"offset": 62367842304,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 62367866880,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.44.ffn_gate.0.weight",
"offset": 62367891456,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.44.ffn_down.0.weight",
"offset": 62424514560,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.0.weight",
"offset": 62481137664,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.1.weight",
"offset": 62537760768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.1.weight",
"offset": 62594383872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate_inp.weight",
"offset": 62651006976,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.44.attn_k.weight",
"offset": 62651105280,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.44.attn_output.weight",
"offset": 62657789952,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.44.attn_q.weight",
"offset": 62679023616,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.44.attn_v.weight",
"offset": 62700257280,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.44.ffn_up.1.weight",
"offset": 62706941952,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.44.ffn_gate.2.weight",
"offset": 62763565056,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.2.weight",
"offset": 62820188160,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.2.weight",
"offset": 62876811264,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.3.weight",
"offset": 62933434368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.3.weight",
"offset": 62990057472,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.3.weight",
"offset": 63046680576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.4.weight",
"offset": 63103303680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.4.weight",
"offset": 63159926784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.4.weight",
"offset": 63216549888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.5.weight",
"offset": 63273172992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.5.weight",
"offset": 63329796096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.5.weight",
"offset": 63386419200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.6.weight",
"offset": 63443042304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.6.weight",
"offset": 63499665408,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.6.weight",
"offset": 63556288512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_gate.7.weight",
"offset": 63612911616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_down.7.weight",
"offset": 63669534720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.ffn_up.7.weight",
"offset": 63726157824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.44.attn_norm.weight",
"offset": 63782780928,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 63782805504,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.45.ffn_gate.0.weight",
"offset": 63782830080,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.45.ffn_down.0.weight",
"offset": 63839453184,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.0.weight",
"offset": 63896076288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.1.weight",
"offset": 63952699392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate_inp.weight",
"offset": 64009322496,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.45.attn_k.weight",
"offset": 64009420800,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.45.attn_output.weight",
"offset": 64016105472,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.45.attn_q.weight",
"offset": 64037339136,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.45.attn_v.weight",
"offset": 64058572800,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.45.ffn_down.1.weight",
"offset": 64065257472,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.45.ffn_up.1.weight",
"offset": 64121880576,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.2.weight",
"offset": 64178503680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.2.weight",
"offset": 64235126784,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.2.weight",
"offset": 64291749888,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.3.weight",
"offset": 64348372992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.3.weight",
"offset": 64404996096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.3.weight",
"offset": 64461619200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.4.weight",
"offset": 64518242304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.4.weight",
"offset": 64574865408,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.4.weight",
"offset": 64631488512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.5.weight",
"offset": 64688111616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.5.weight",
"offset": 64744734720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.5.weight",
"offset": 64801357824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.6.weight",
"offset": 64857980928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.6.weight",
"offset": 64914604032,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.6.weight",
"offset": 64971227136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_gate.7.weight",
"offset": 65027850240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_down.7.weight",
"offset": 65084473344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.ffn_up.7.weight",
"offset": 65141096448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.45.attn_norm.weight",
"offset": 65197719552,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 65197744128,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.46.ffn_gate.0.weight",
"offset": 65197768704,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.46.ffn_down.0.weight",
"offset": 65254391808,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.0.weight",
"offset": 65311014912,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "\u0000\u0000\u0000blk.50.ffn_down.2.weigh",
"offset": 65367638016,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.46.attn_k.weight",
"offset": 65367736320,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.46.attn_output.weight",
"offset": 65374420992,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.46.attn_q.weight",
"offset": 65395654656,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.46.attn_v.weight",
"offset": 65416888320,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.46.ffn_gate.1.weight",
"offset": 65423572992,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.46.ffn_down.1.weight",
"offset": 65480196096,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.1.weight",
"offset": 65536819200,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.2.weight",
"offset": 65593442304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.2.weight",
"offset": 65650065408,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.2.weight",
"offset": 65706688512,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.3.weight",
"offset": 65763311616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.3.weight",
"offset": 65819934720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.3.weight",
"offset": 65876557824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.4.weight",
"offset": 65933180928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.4.weight",
"offset": 65989804032,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.4.weight",
"offset": 66046427136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.5.weight",
"offset": 66103050240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.5.weight",
"offset": 66159673344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.5.weight",
"offset": 66216296448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.6.weight",
"offset": 66272919552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.6.weight",
"offset": 66329542656,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.6.weight",
"offset": 66386165760,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_gate.7.weight",
"offset": 66442788864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_down.7.weight",
"offset": 66499411968,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.ffn_up.7.weight",
"offset": 66556035072,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.46.attn_norm.weight",
"offset": 66612658176,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 66612682752,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.47.ffn_gate.0.weight",
"offset": 66612707328,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.47.ffn_down.0.weight",
"offset": 66669330432,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate_inp.weight",
"offset": 66725953536,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.47.attn_k.weight",
"offset": 66726051840,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.47.attn_output.weight",
"offset": 66732736512,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.47.attn_q.weight",
"offset": 66753970176,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.47.attn_v.weight",
"offset": 66775203840,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.47.ffn_up.0.weight",
"offset": 66781888512,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.47.ffn_gate.1.weight",
"offset": 66838511616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.1.weight",
"offset": 66895134720,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.1.weight",
"offset": 66951757824,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.2.weight",
"offset": 67008380928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.2.weight",
"offset": 67065004032,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.2.weight",
"offset": 67121627136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.3.weight",
"offset": 67178250240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.3.weight",
"offset": 67234873344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.3.weight",
"offset": 67291496448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.4.weight",
"offset": 67348119552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.4.weight",
"offset": 67404742656,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.4.weight",
"offset": 67461365760,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.5.weight",
"offset": 67517988864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.5.weight",
"offset": 67574611968,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.5.weight",
"offset": 67631235072,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.6.weight",
"offset": 67687858176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.6.weight",
"offset": 67744481280,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.6.weight",
"offset": 67801104384,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_gate.7.weight",
"offset": 67857727488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_down.7.weight",
"offset": 67914350592,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.ffn_up.7.weight",
"offset": 67970973696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.47.attn_norm.weight",
"offset": 68027596800,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 68027621376,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.48.ffn_gate.0.weight",
"offset": 68027645952,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.48.ffn_gate_inp.weight",
"offset": 68084269056,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.48.attn_k.weight",
"offset": 68084367360,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.48.attn_output.weight",
"offset": 68091052032,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.48.attn_q.weight",
"offset": 68112285696,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.48.attn_v.weight",
"offset": 68133519360,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.48.ffn_down.0.weight",
"offset": 68140204032,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.48.ffn_up.0.weight",
"offset": 68196827136,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.1.weight",
"offset": 68253450240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.1.weight",
"offset": 68310073344,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.1.weight",
"offset": 68366696448,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.2.weight",
"offset": 68423319552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.2.weight",
"offset": 68479942656,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.2.weight",
"offset": 68536565760,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.3.weight",
"offset": 68593188864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.3.weight",
"offset": 68649811968,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.3.weight",
"offset": 68706435072,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.4.weight",
"offset": 68763058176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.4.weight",
"offset": 68819681280,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.4.weight",
"offset": 68876304384,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.5.weight",
"offset": 68932927488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.5.weight",
"offset": 68989550592,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.5.weight",
"offset": 69046173696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.6.weight",
"offset": 69102796800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.6.weight",
"offset": 69159419904,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.6.weight",
"offset": 69216043008,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_gate.7.weight",
"offset": 69272666112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_down.7.weight",
"offset": 69329289216,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.ffn_up.7.weight",
"offset": 69385912320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.48.attn_norm.weight",
"offset": 69442535424,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 69442560000,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.49.ffn_gate_inp.weight",
"offset": 69442584576,
"shape": [
6144,
8
],
"size": 24576,
"type": 1
},
{
"name": "blk.49.attn_k.weight",
"offset": 69442682880,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.49.attn_output.weight",
"offset": 69449367552,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.49.attn_q.weight",
"offset": 69470601216,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.49.attn_v.weight",
"offset": 69491834880,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.49.ffn_gate.0.weight",
"offset": 69498519552,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.49.ffn_down.0.weight",
"offset": 69555142656,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.0.weight",
"offset": 69611765760,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.1.weight",
"offset": 69668388864,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.1.weight",
"offset": 69725011968,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.1.weight",
"offset": 69781635072,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.2.weight",
"offset": 69838258176,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.2.weight",
"offset": 69894881280,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.2.weight",
"offset": 69951504384,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.3.weight",
"offset": 70008127488,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.3.weight",
"offset": 70064750592,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.3.weight",
"offset": 70121373696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.4.weight",
"offset": 70177996800,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.4.weight",
"offset": 70234619904,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.4.weight",
"offset": 70291243008,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.5.weight",
"offset": 70347866112,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.5.weight",
"offset": 70404489216,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.5.weight",
"offset": 70461112320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.6.weight",
"offset": 70517735424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.6.weight",
"offset": 70574358528,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.6.weight",
"offset": 70630981632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_gate.7.weight",
"offset": 70687604736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_down.7.weight",
"offset": 70744227840,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.ffn_up.7.weight",
"offset": 70800850944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.49.attn_norm.weight",
"offset": 70857474048,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 70857498624,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 70857523200,
"shape": [
6144,
1024
],
"size": 24576,
"type": 8
},
{
"name": "blk.50.attn_q.weight",
"offset": 70864207872,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.50.attn_v.weight",
"offset": 70885441536,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.50.ffn_gate.0.weight",
"offset": 70892126208,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.50.ffn_down.0.weight",
"offset": 70948749312,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.0.weight",
"offset": 71005372416,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.1.weight",
"offset": 71061995520,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.1.weight",
"offset": 71118618624,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.1.weight",
"offset": 71175241728,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.2.weight",
"offset": 71231864832,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\ufffd\u0006\u0012\u0000\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl",
"offset": 71288487936,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.2.weight",
"offset": 71345111040,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.3.weight",
"offset": 71401734144,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.3.weight",
"offset": 71458357248,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.3.weight",
"offset": 71514980352,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.4.weight",
"offset": 71571603456,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.4.weight",
"offset": 71628226560,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.4.weight",
"offset": 71684849664,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.5.weight",
"offset": 71741472768,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.5.weight",
"offset": 71798095872,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.5.weight",
"offset": 71854718976,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.6.weight",
"offset": 71911342080,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.6.weight",
"offset": 71967965184,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.6.weight",
"offset": 72024588288,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate.7.weight",
"offset": 72081211392,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_down.7.weight",
"offset": 72137834496,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_up.7.weight",
"offset": 72194457600,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.50.ffn_gate_inp.weight",
"offset": 72251080704,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.50.attn_norm.weight",
"offset": 72251179008,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 72251203584,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.50.attn_output.weight",
"offset": 72251228160,
"shape": [
6144,
6144
],
"size": 24576,
"type": 2
},
{
"name": "blk.51.attn_k.weight",
"offset": 72272461824,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.51.attn_q.weight",
"offset": 72279146496,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.51.ffn_gate.0.weight",
"offset": 72300380160,
"shape": [
6144,
16384
],
"size": 21233664,
"type": 2
},
{
"name": "blk.51.ffn_down.0.weight",
"offset": 72357003264,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.0.weight",
"offset": 72413626368,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.1.weight",
"offset": 72470249472,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.1.weight",
"offset": 72526872576,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.1.weight",
"offset": 72583495680,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.2.weight",
"offset": 72640118784,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.2.weight",
"offset": 72696741888,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.2.weight",
"offset": 72753364992,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.3.weight",
"offset": 72809988096,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.3.weight",
"offset": 72866611200,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.3.weight",
"offset": 72923234304,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.4.weight",
"offset": 72979857408,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.4.weight",
"offset": 73036480512,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.4.weight",
"offset": 73093103616,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.5.weight",
"offset": 73149726720,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.5.weight",
"offset": 73206349824,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.5.weight",
"offset": 73262972928,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.6.weight",
"offset": 73319596032,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.6.weight",
"offset": 73376219136,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.6.weight",
"offset": 73432842240,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate.7.weight",
"offset": 73489465344,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_down.7.weight",
"offset": 73546088448,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_up.7.weight",
"offset": 73602711552,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.51.ffn_gate_inp.weight",
"offset": 73659334656,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.51.attn_norm.weight",
"offset": 73659432960,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 73659457536,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.51.attn_output.weight",
"offset": 73659482112,
"shape": [
6144,
6144
],
"size": 24576,
"type": 2
},
{
"name": "blk.51.attn_v.weight",
"offset": 73680715776,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.52.attn_q.weight",
"offset": 73687400448,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.52.ffn_gate.0.weight",
"offset": 73708634112,
"shape": [
6144,
16384
],
"size": 21233664,
"type": 2
},
{
"name": "blk.52.ffn_down.0.weight",
"offset": 73765257216,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.0.weight",
"offset": 73821880320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.1.weight",
"offset": 73878503424,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.1.weight",
"offset": 73935126528,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.1.weight",
"offset": 73991749632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.2.weight",
"offset": 74048372736,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.2.weight",
"offset": 74104995840,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.2.weight",
"offset": 74161618944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.3.weight",
"offset": 74218242048,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.3.weight",
"offset": 74274865152,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.3.weight",
"offset": 74331488256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.4.weight",
"offset": 74388111360,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.4.weight",
"offset": 74444734464,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.4.weight",
"offset": 74501357568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.5.weight",
"offset": 74557980672,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.5.weight",
"offset": 74614603776,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.5.weight",
"offset": 74671226880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.6.weight",
"offset": 74727849984,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.6.weight",
"offset": 74784473088,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.6.weight",
"offset": 74841096192,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate.7.weight",
"offset": 74897719296,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_down.7.weight",
"offset": 74954342400,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_up.7.weight",
"offset": 75010965504,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.52.ffn_gate_inp.weight",
"offset": 75067588608,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.52.attn_norm.weight",
"offset": 75067686912,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 75067711488,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 75067736064,
"shape": [
6144,
1024
],
"size": 24576,
"type": 8
},
{
"name": "blk.52.attn_output.weight",
"offset": 75074420736,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.52.attn_v.weight",
"offset": 75095654400,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.53.ffn_gate.0.weight",
"offset": 75102339072,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.53.ffn_down.0.weight",
"offset": 75158962176,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.0.weight",
"offset": 75215585280,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.1.weight",
"offset": 75272208384,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.1.weight",
"offset": 75328831488,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.1.weight",
"offset": 75385454592,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.2.weight",
"offset": 75442077696,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.2.weight",
"offset": 75498700800,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.2.weight",
"offset": 75555323904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.3.weight",
"offset": 75611947008,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.3.weight",
"offset": 75668570112,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.3.weight",
"offset": 75725193216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.4.weight",
"offset": 75781816320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.4.weight",
"offset": 75838439424,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.4.weight",
"offset": 75895062528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.5.weight",
"offset": 75951685632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.5.weight",
"offset": 76008308736,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.5.weight",
"offset": 76064931840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.6.weight",
"offset": 76121554944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.6.weight",
"offset": 76178178048,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_up.6.weight",
"offset": 76234801152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate.7.weight",
"offset": 76291424256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_down.7.weight",
"offset": 76348047360,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.53.ffn_gate_inp.weight",
"offset": 76404670464,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.53.attn_k.weight",
"offset": 76404768768,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.53.attn_output.weight",
"offset": 76411453440,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.53.attn_q.weight",
"offset": 76432687104,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.53.attn_v.weight",
"offset": 76453920768,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.53.ffn_up.7.weight",
"offset": 76460605440,
"shape": [
6144,
16384
],
"size": 6684672,
"type": 2
},
{
"name": "blk.53.attn_norm.weight",
"offset": 76517228544,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 76517253120,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.54.ffn_gate.0.weight",
"offset": 76517277696,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.54.ffn_down.0.weight",
"offset": 76573900800,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.0.weight",
"offset": 76630523904,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.1.weight",
"offset": 76687147008,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.1.weight",
"offset": 76743770112,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.1.weight",
"offset": 76800393216,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.2.weight",
"offset": 76857016320,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.2.weight",
"offset": 76913639424,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.2.weight",
"offset": 76970262528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.3.weight",
"offset": 77026885632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.3.weight",
"offset": 77083508736,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.3.weight",
"offset": 77140131840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.4.weight",
"offset": 77196754944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.4.weight",
"offset": 77253378048,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.4.weight",
"offset": 77310001152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.5.weight",
"offset": 77366624256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.5.weight",
"offset": 77423247360,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.5.weight",
"offset": 77479870464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.6.weight",
"offset": 77536493568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_down.6.weight",
"offset": 77593116672,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_up.6.weight",
"offset": 77649739776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate.7.weight",
"offset": 77706362880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.ffn_gate_inp.weight",
"offset": 77762985984,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.54.attn_k.weight",
"offset": 77763084288,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.54.attn_output.weight",
"offset": 77769768960,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.54.attn_q.weight",
"offset": 77791002624,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.54.attn_v.weight",
"offset": 77812236288,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "blk.54.ffn_down.7.weight",
"offset": 77818920960,
"shape": [
16384,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.54.ffn_up.7.weight",
"offset": 77875544064,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.54.attn_norm.weight",
"offset": 77932167168,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 77932191744,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.55.ffn_gate.0.weight",
"offset": 77932216320,
"shape": [
6144,
16384
],
"size": 24576,
"type": 2
},
{
"name": "blk.55.ffn_down.0.weight",
"offset": 77988839424,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.0.weight",
"offset": 78045462528,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.1.weight",
"offset": 78102085632,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.1.weight",
"offset": 78158708736,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.1.weight",
"offset": 78215331840,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.2.weight",
"offset": 78271954944,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.2.weight",
"offset": 78328578048,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.2.weight",
"offset": 78385201152,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.3.weight",
"offset": 78441824256,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.3.weight",
"offset": 78498447360,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.3.weight",
"offset": 78555070464,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.4.weight",
"offset": 78611693568,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.4.weight",
"offset": 78668316672,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.4.weight",
"offset": 78724939776,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.5.weight",
"offset": 78781562880,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.5.weight",
"offset": 78838185984,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.5.weight",
"offset": 78894809088,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate.6.weight",
"offset": 78951432192,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_down.6.weight",
"offset": 79008055296,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.6.weight",
"offset": 79064678400,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_gate_inp.weight",
"offset": 79121301504,
"shape": [
6144,
8
],
"size": 56623104,
"type": 1
},
{
"name": "blk.55.attn_k.weight",
"offset": 79121399808,
"shape": [
6144,
1024
],
"size": 98304,
"type": 8
},
{
"name": "blk.55.attn_output.weight",
"offset": 79128084480,
"shape": [
6144,
6144
],
"size": 6684672,
"type": 2
},
{
"name": "blk.55.attn_q.weight",
"offset": 79149318144,
"shape": [
6144,
6144
],
"size": 21233664,
"type": 2
},
{
"name": "blk.55.attn_v.weight",
"offset": 79170551808,
"shape": [
6144,
1024
],
"size": 21233664,
"type": 8
},
{
"name": "output.weight",
"offset": 79177236480,
"shape": [
6144,
32000
],
"size": 6684672,
"type": 14
},
{
"name": "blk.55.ffn_gate.7.weight",
"offset": 79338516480,
"shape": [
6144,
16384
],
"size": 161280000,
"type": 2
},
{
"name": "blk.55.ffn_down.7.weight",
"offset": 79395139584,
"shape": [
16384,
6144
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.ffn_up.7.weight",
"offset": 79451762688,
"shape": [
6144,
16384
],
"size": 56623104,
"type": 2
},
{
"name": "blk.55.attn_norm.weight",
"offset": 79508385792,
"shape": [
6144
],
"size": 56623104,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 79508410368,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 79508434944,
"shape": [
6144
],
"size": 24576,
"type": 0
}
],
"version": 3
}