Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
e268c9a408ce · 21GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "Yi-1.5-34B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 258048000,
"shape": [
7168
],
"size": 258048000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 258076672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 378499072,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 461074432,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 543649792,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 543678464,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 547807232,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 576708608,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 605609984,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 611631104,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 611659776,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 732082176,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 814657536,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 897232896,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 897261568,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 901390336,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 930291712,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 959193088,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 965214208,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 965242880,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1085665280,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1168240640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1250816000,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1250844672,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 1254973440,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 1283874816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 1312776192,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1318797312,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.3.attn_k.weight",
"offset": 1401372672,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 1405501440,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 1434402816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 1463304192,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1469325312,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1469353984,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1589776384,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1672351744,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1672380416,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1672409088,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1792831488,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1875406848,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1957982208,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1958010880,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 1962139648,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 1991041024,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 2019942400,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2025963520,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2025992192,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 2146414592,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2228989952,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2311565312,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 2311593984,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 2315722752,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 2344624128,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 2373525504,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 2379546624,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 2379575296,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2499997696,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2582573056,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2665148416,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2665177088,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 2669305856,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 2698207232,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 2727108608,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2733129728,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2815705088,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.7.attn_k.weight",
"offset": 2898280448,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 2902409216,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 2931310592,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 2960211968,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2964340736,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2964369408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 3046944768,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 3129520128,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 3212095488,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 3212124160,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 3216252928,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 3245154304,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 3274055680,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 3278184448,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 3278213120,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 3360788480,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3443363840,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 3525939200,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 3525967872,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 3530096640,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 3558998016,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 3587899392,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 3593920512,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 3598049280,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 3626950656,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 3655852032,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 3659980800,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 3660009472,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 3780431872,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 3780460544,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 3780489216,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 3863064576,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 3945639936,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4028215296,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 4028243968,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 4032372736,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 4061274112,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 4090175488,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 4094304256,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 4094332928,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 4176908288,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4259483648,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 4342059008,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 4342087680,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 4346216448,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 4375117824,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 4404019200,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 4410040320,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 4410068992,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 4530491392,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 4613066752,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 4695642112,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 4695670784,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 4695699456,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 4778274816,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 4860850176,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 4943425536,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 4943454208,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 4947582976,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 4976484352,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 5005385728,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5009514496,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 5009543168,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 5092118528,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 5174693888,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 5257269248,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 5257297920,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 5261426688,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 5290328064,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 5319229440,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 5323358208,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 5323386880,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 5443809280,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 5526384640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 5608960000,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 5608988672,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 5613117440,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 5642018816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 5670920192,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 5676941312,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.16.attn_k.weight",
"offset": 5759516672,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 5763645440,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 5792546816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 5821448192,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 5825576960,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 5825605632,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 5908180992,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 5990756352,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 5990785024,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 5990813696,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 6073389056,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 6155964416,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 6238539776,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 6238568448,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 6242697216,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 6271598592,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 6300499968,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 6304628736,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 6304657408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 6425079808,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 6507655168,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 6590230528,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 6590259200,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 6594387968,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 6623289344,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 6652190720,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 6658211840,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 6658240512,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 6740815872,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 6823391232,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 6905966592,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 6905995264,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 6910124032,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 6939025408,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 6967926784,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 6972055552,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 7054630912,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.20.attn_k.weight",
"offset": 7137206272,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 7141335040,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 7170236416,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 7199137792,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 7203266560,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 7203295232,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 7285870592,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 7285899264,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 7285927936,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 7406350336,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 7488925696,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 7571501056,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 7571529728,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 7575658496,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 7604559872,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 7633461248,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 7639482368,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 7639511040,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 7722086400,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 7804661760,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 7887237120,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 7887265792,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 7891394560,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 7920295936,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 7949197312,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 7953326080,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 7953354752,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 8035930112,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 8118505472,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 8201080832,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 8201109504,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 8205238272,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 8234139648,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 8263041024,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 8267169792,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 8267198464,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 8387620864,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 8470196224,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 8552771584,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 8552800256,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 8556929024,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "k.40.attn_norm.weigh",
"offset": 8585830400,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 8614731776,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.25.attn_k.weight",
"offset": 8620752896,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 8624881664,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 8653783040,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 8682684416,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 8686813184,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 8686841856,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 8769417216,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 8851992576,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 8934567936,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 8934596608,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 8934625280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 9017200640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 9099776000,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 9182351360,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 9182380032,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 9186508800,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 9215410176,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 9244311552,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 9248440320,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 9248468992,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 9368891392,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 9451466752,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 9534042112,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 9534070784,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 9538199552,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 9567100928,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 9596002304,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 9602023424,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 9602052096,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 9684627456,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 9767202816,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 9849778176,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 9849806848,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 9853935616,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 9882836992,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 9911738368,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 9915867136,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 12
},
{
"name": "blk.29.attn_k.weight",
"offset": 9998442496,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 10002571264,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 10031472640,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 10060374016,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 10064502784,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 10064531456,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 10147106816,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 10229682176,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 10229710848,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 10229739520,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 10350161920,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 10432737280,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 10515312640,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 10515341312,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 10519470080,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 10548371456,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 10577272832,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 10583293952,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 10583322624,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 10665897984,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 10748473344,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 10831048704,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 10831077376,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 10835206144,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 10864107520,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 10893008896,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 10897137664,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 10897166336,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 10979741696,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.32.ffn_up.weight",
"offset": 11062317056,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 11144892416,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 11144921088,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.32.attn_output.weight",
"offset": 11149049856,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.32.attn_q.weight",
"offset": 11177951232,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.32.attn_v.weight",
"offset": 11206852608,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 11210981376,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 12
},
{
"name": "blk.33.ffn_up.weight",
"offset": 11293556736,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.33.attn_k.weight",
"offset": 11376132096,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.33.attn_output.weight",
"offset": 11380260864,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.33.attn_q.weight",
"offset": 11409162240,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.33.attn_v.weight",
"offset": 11438063616,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 11444084736,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 11444113408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 11564535808,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 11564564480,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 11564593152,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 11647168512,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.34.ffn_up.weight",
"offset": 11729743872,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 11812319232,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 11812347904,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.34.attn_output.weight",
"offset": 11816476672,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.34.attn_q.weight",
"offset": 11845378048,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.34.attn_v.weight",
"offset": 11874279424,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 11878408192,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 11878436864,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 11961012224,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.35.ffn_up.weight",
"offset": 12043587584,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 12126162944,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 12126191616,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.35.attn_output.weight",
"offset": 12130320384,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.35.attn_q.weight",
"offset": 12159221760,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.35.attn_v.weight",
"offset": 12188123136,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.36.attn_norm.weight",
"offset": 12192251904,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 12192280576,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 12312702976,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.36.ffn_up.weight",
"offset": 12395278336,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 12477853696,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 12477882368,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.36.attn_output.weight",
"offset": 12482011136,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.36.attn_q.weight",
"offset": 12510912512,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.36.attn_v.weight",
"offset": 12539813888,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 12545835008,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 12545863680,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 12628439040,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.37.ffn_up.weight",
"offset": 12711014400,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 12793589760,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 12793618432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.37.attn_output.weight",
"offset": 12797747200,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.37.attn_q.weight",
"offset": 12826648576,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.37.attn_v.weight",
"offset": 12855549952,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.38.attn_k.weight",
"offset": 12859678720,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 12
},
{
"name": "blk.38.attn_output.weight",
"offset": 12863807488,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.38.attn_q.weight",
"offset": 12892708864,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.38.attn_v.weight",
"offset": 12921610240,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 12925739008,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 12925767680,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 13008343040,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.38.ffn_up.weight",
"offset": 13090918400,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 13173493760,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 13173522432,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 13173551104,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 13293973504,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.39.ffn_up.weight",
"offset": 13376548864,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 13459124224,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 13459152896,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.39.attn_output.weight",
"offset": 13463281664,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.39.attn_q.weight",
"offset": 13492183040,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.39.attn_v.weight",
"offset": 13521084416,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 13527105536,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 13527134208,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 13609709568,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.40.ffn_up.weight",
"offset": 13692284928,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 13774860288,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 13774888960,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.40.attn_output.weight",
"offset": 13779017728,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.40.attn_q.weight",
"offset": 13807919104,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.40.attn_v.weight",
"offset": 13836820480,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.41.attn_norm.weight",
"offset": 13840949248,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 13840977920,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 13923553280,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.41.ffn_up.weight",
"offset": 14006128640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 14088704000,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 14088732672,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.41.attn_output.weight",
"offset": 14092861440,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.41.attn_q.weight",
"offset": 14121762816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.41.attn_v.weight",
"offset": 14150664192,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 14154792960,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 12
},
{
"name": "blk.42.attn_k.weight",
"offset": 14237368320,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.42.attn_output.weight",
"offset": 14241497088,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.42.attn_q.weight",
"offset": 14270398464,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.42.attn_v.weight",
"offset": 14299299840,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 14305320960,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 14305349632,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 14425772032,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 14508347392,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 14508376064,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 14508404736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 14590980096,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.43.ffn_up.weight",
"offset": 14673555456,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 14756130816,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 14756159488,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.43.attn_output.weight",
"offset": 14760288256,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.43.attn_q.weight",
"offset": 14789189632,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.43.attn_v.weight",
"offset": 14818091008,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.44.attn_norm.weight",
"offset": 14822219776,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 14822248448,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 14904823808,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.44.ffn_up.weight",
"offset": 14987399168,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 15069974528,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 15070003200,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.44.attn_output.weight",
"offset": 15074131968,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.44.attn_q.weight",
"offset": 15103033344,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.44.attn_v.weight",
"offset": 15131934720,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.45.attn_norm.weight",
"offset": 15136063488,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 15136092160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 15256514560,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.45.ffn_up.weight",
"offset": 15339089920,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 15421665280,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 15421693952,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.45.attn_output.weight",
"offset": 15425822720,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.45.attn_q.weight",
"offset": 15454724096,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.45.attn_v.weight",
"offset": 15483625472,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 15489646592,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.46.ffn_up.weight",
"offset": 15572221952,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.46.attn_k.weight",
"offset": 15654797312,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.46.attn_output.weight",
"offset": 15658926080,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.46.attn_q.weight",
"offset": 15687827456,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.46.attn_v.weight",
"offset": 15716728832,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.46.attn_norm.weight",
"offset": 15720857600,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 15720886272,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 15803461632,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 15803490304,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 15803518976,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 15886094336,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.47.ffn_up.weight",
"offset": 15968669696,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 16051245056,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 16051273728,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "\u0000\u0000\u0000\ufffdT\u0004\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.55.a",
"offset": 16055402496,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.47.attn_q.weight",
"offset": 16084303872,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.47.attn_v.weight",
"offset": 16113205248,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.48.attn_norm.weight",
"offset": 16117334016,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 16117362688,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 16237785088,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.48.ffn_up.weight",
"offset": 16320360448,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 16402935808,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 16402964480,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.48.attn_output.weight",
"offset": 16407093248,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.48.attn_q.weight",
"offset": 16435994624,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.48.attn_v.weight",
"offset": 16464896000,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.49.attn_norm.weight",
"offset": 16470917120,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 16470945792,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 16553521152,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.49.ffn_up.weight",
"offset": 16636096512,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 16718671872,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 16718700544,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.49.attn_output.weight",
"offset": 16722829312,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.49.attn_q.weight",
"offset": 16751730688,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.49.attn_v.weight",
"offset": 16780632064,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.50.attn_norm.weight",
"offset": 16784760832,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 16784789504,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 16867364864,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.50.ffn_up.weight",
"offset": 16949940224,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 17032515584,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 17032544256,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.50.attn_output.weight",
"offset": 17036673024,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.50.attn_q.weight",
"offset": 17065574400,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.50.attn_v.weight",
"offset": 17094475776,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 12
},
{
"name": "blk.51.attn_k.weight",
"offset": 17098604544,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 12
},
{
"name": "blk.51.attn_output.weight",
"offset": 17102733312,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.51.attn_q.weight",
"offset": 17131634688,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.51.attn_v.weight",
"offset": 17160536064,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 17166557184,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 17166585856,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 17287008256,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.51.ffn_up.weight",
"offset": 17369583616,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 17452158976,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 17452187648,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 17452216320,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 17572638720,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.52.ffn_up.weight",
"offset": 17655214080,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 17737789440,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 17737818112,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.52.attn_output.weight",
"offset": 17741946880,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.52.attn_q.weight",
"offset": 17770848256,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.52.attn_v.weight",
"offset": 17799749632,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 17805770752,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 17805799424,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 17926221824,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.53.ffn_up.weight",
"offset": 18008797184,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 18091372544,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 18091401216,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.53.attn_output.weight",
"offset": 18095529984,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.53.attn_q.weight",
"offset": 18124431360,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.53.attn_v.weight",
"offset": 18153332736,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 18159353856,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 18159382528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 18279804928,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.54.ffn_up.weight",
"offset": 18362380288,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 18444955648,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 18444984320,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.54.attn_output.weight",
"offset": 18449113088,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.54.attn_q.weight",
"offset": 18478014464,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.54.attn_v.weight",
"offset": 18506915840,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 18512936960,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.55.attn_k.weight",
"offset": 18595512320,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.55.attn_output.weight",
"offset": 18599641088,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.55.attn_q.weight",
"offset": 18628542464,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.55.attn_v.weight",
"offset": 18657443840,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.55.attn_norm.weight",
"offset": 18663464960,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 18663493632,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.55.ffn_up.weight",
"offset": 18783916032,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 18866491392,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 18866520064,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 18866548736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 18986971136,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.56.ffn_up.weight",
"offset": 19069546496,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 19152121856,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 19152150528,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.56.attn_output.weight",
"offset": 19156279296,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.56.attn_q.weight",
"offset": 19185180672,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.56.attn_v.weight",
"offset": 19214082048,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 19220103168,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 19220131840,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 19340554240,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.57.ffn_up.weight",
"offset": 19423129600,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 19505704960,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 19505733632,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.57.attn_output.weight",
"offset": 19509862400,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.57.attn_q.weight",
"offset": 19538763776,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.57.attn_v.weight",
"offset": 19567665152,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.58.attn_norm.weight",
"offset": 19573686272,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 19573714944,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 19694137344,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 12
},
{
"name": "blk.58.ffn_up.weight",
"offset": 19776712704,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 19859288064,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 19859316736,
"shape": [
7168,
1024
],
"size": 28672,
"type": 12
},
{
"name": "blk.58.attn_output.weight",
"offset": 19863445504,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.58.attn_q.weight",
"offset": 19892346880,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.58.attn_v.weight",
"offset": 19921248256,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 19927269376,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 12
},
{
"name": "blk.59.ffn_up.weight",
"offset": 20009844736,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 12
},
{
"name": "blk.59.attn_k.weight",
"offset": 20092420096,
"shape": [
7168,
1024
],
"size": 82575360,
"type": 12
},
{
"name": "blk.59.attn_output.weight",
"offset": 20096548864,
"shape": [
7168,
7168
],
"size": 4128768,
"type": 12
},
{
"name": "blk.59.attn_q.weight",
"offset": 20125450240,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 12
},
{
"name": "blk.59.attn_v.weight",
"offset": 20154351616,
"shape": [
7168,
1024
],
"size": 28901376,
"type": 14
},
{
"name": "output.weight",
"offset": 20160372736,
"shape": [
7168,
64000
],
"size": 6021120,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 20536692736,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 20536721408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 20657143808,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 20657172480,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}