Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
6c57235c9130 · 7.2GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": "Yi-1.5-9B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 4,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 48,
"llama.context_length": 16384,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 11008,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 8829407232,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 215040000,
"shape": [
4096
],
"size": 215040000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 215056384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 252043264,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 289030144,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 326017024,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 326033408,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 327753728,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.0.attn_q.weight",
"offset": 341516288,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 355278848,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 356999168,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 357015552,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 394002432,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 430989312,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 467976192,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 467992576,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 469712896,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.1.attn_q.weight",
"offset": 483475456,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 497238016,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 498958336,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 498974720,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 535961600,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 572948480,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 609935360,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 609951744,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 611672064,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.10.attn_q.weight",
"offset": 625434624,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 639197184,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 640917504,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 640933888,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 677920768,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 714907648,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 751894528,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 751910912,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 753631232,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.11.attn_q.weight",
"offset": 767393792,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 781156352,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 782876672,
"shape": [
4096,
11008
],
"size": 1720320,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 819863552,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 856850432,
"shape": [
4096,
512
],
"size": 36986880,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 858570752,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.12.attn_q.weight",
"offset": 872333312,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 886095872,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 887816192,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 887832576,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 924819456,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 961806336,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 998793216,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 998809600,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 1000529920,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.2.attn_q.weight",
"offset": 1014292480,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 1028055040,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1029775360,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1029791744,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1066778624,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1103765504,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1140752384,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 1140768768,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 1142489088,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.3.attn_q.weight",
"offset": 1156251648,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 1170014208,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1171734528,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1171750912,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1208737792,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1245724672,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1282711552,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1282727936,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 1284448256,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.4.attn_q.weight",
"offset": 1298210816,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 1311973376,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1313693696,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1313710080,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1350696960,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1387683840,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1424670720,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1424687104,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 1426407424,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.5.attn_q.weight",
"offset": 1440169984,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 1453932544,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1455652864,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1455669248,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1492656128,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1529643008,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1566629888,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1566646272,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 1568366592,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.6.attn_q.weight",
"offset": 1582129152,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 1595891712,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1597612032,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1597628416,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1634615296,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1671602176,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1708589056,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1708605440,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 1710325760,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.7.attn_q.weight",
"offset": 1724088320,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 1737850880,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1739571200,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1739587584,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1776574464,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1813561344,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1850548224,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1850564608,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 1852284928,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.8.attn_q.weight",
"offset": 1866047488,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 1879810048,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 1881530368,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 1881546752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 1918533632,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 1955520512,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 1992507392,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 1992523776,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 1994244096,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.9.attn_q.weight",
"offset": 2008006656,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 2021769216,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2023489536,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2023505920,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2060492800,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2060509184,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2060525568,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2097512448,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2134499328,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2171486208,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2171502592,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 2173222912,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.13.attn_q.weight",
"offset": 2186985472,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 2200748032,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 2202468352,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 2202484736,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 2239471616,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 2276458496,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 2313445376,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 2313461760,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 2315182080,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.14.attn_q.weight",
"offset": 2328944640,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 2342707200,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2344427520,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2344443904,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2381430784,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2418417664,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2455404544,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2455420928,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 2457141248,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.15.attn_q.weight",
"offset": 2470903808,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.15.attn_v.weight",
"offset": 2484666368,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2486386688,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2486403072,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2523389952,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2560376832,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2597363712,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2597380096,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 2599100416,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.16.attn_q.weight",
"offset": 2612862976,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 2626625536,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2628345856,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2628362240,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2665349120,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2702336000,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2739322880,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2739339264,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 2741059584,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.17.attn_q.weight",
"offset": 2754822144,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 2768584704,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2770305024,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2770321408,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2807308288,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2844295168,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2881282048,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2881298432,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 2883018752,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.18.attn_q.weight",
"offset": 2896781312,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 2910543872,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2912264192,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2912280576,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2949267456,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2986254336,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 3023241216,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 3023257600,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 3024977920,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.19.attn_q.weight",
"offset": 3038740480,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 3052503040,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 3054223360,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 3054239744,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 3091226624,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 3128213504,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 3165200384,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 3165216768,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 3166937088,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.20.attn_q.weight",
"offset": 3180699648,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 3194462208,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 3196182528,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 3196198912,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 3233185792,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 3270172672,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 3307159552,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 3307175936,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 3308896256,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.21.attn_q.weight",
"offset": 3322658816,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 3336421376,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 3338141696,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 3338158080,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 3375144960,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 3412131840,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 3449118720,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 3449135104,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 3450855424,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.22.attn_q.weight",
"offset": 3464617984,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 3478380544,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 3480100864,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 3480117248,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3517104128,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3554091008,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 3591077888,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 3591094272,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 3592814592,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.23.attn_q.weight",
"offset": 3606577152,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 3620339712,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 3622060032,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 3622076416,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 3659063296,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3696050176,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3733037056,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3733053440,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 3734773760,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.24.attn_q.weight",
"offset": 3748536320,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 3762298880,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3764019200,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3764035584,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3801022464,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3838009344,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3874996224,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3875012608,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 3876732928,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.25.attn_q.weight",
"offset": 3890495488,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 3904258048,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3905978368,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3905994752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3942981632,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3979968512,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 4016955392,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 4016971776,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 4018692096,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.26.attn_q.weight",
"offset": 4032454656,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 4046217216,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.attn_k.weight",
"offset": 4047937536,
"shape": [
4096,
512
],
"size": 1720320,
"type": 14
},
{
"name": "blk.27.attn_q.weight",
"offset": 4049657856,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 4063420416,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 4065140736,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 4065157120,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 4102144000,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 4139130880,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 4176117760,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 4176134144,
"shape": [
4096,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 4189896704,
"shape": [
4096
],
"size": 13762560,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 4189913088,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 4226899968,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 4263886848,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 4300873728,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 4300890112,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 4302610432,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.28.attn_q.weight",
"offset": 4316372992,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 4330135552,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 4331855872,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 4331872256,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 4368859136,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 4405846016,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 4442832896,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 4442849280,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 4444569600,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.29.attn_q.weight",
"offset": 4458332160,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 4472094720,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 4473815040,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 4473831424,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 4510818304,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 4547805184,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 4584792064,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 4584808448,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 4586528768,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.30.attn_q.weight",
"offset": 4600291328,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 4614053888,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 4615774208,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 4615790592,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 4652777472,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4689764352,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4726751232,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 4726767616,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 4728487936,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.31.attn_q.weight",
"offset": 4742250496,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 4756013056,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.32.attn_norm.weight",
"offset": 4757733376,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 4757749760,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 4794736640,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.32.ffn_up.weight",
"offset": 4831723520,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "\u0000\ufffd2\"\u0001\u0000\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.32",
"offset": 4868710400,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 4868726784,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.32.attn_output.weight",
"offset": 4870447104,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.32.attn_q.weight",
"offset": 4884209664,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.32.attn_v.weight",
"offset": 4897972224,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 4899692544,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 4899708928,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 4936695808,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.33.ffn_up.weight",
"offset": 4973682688,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 5010669568,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 5010685952,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.33.attn_output.weight",
"offset": 5012406272,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.33.attn_q.weight",
"offset": 5026168832,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.33.attn_v.weight",
"offset": 5039931392,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.34.attn_norm.weight",
"offset": 5041651712,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 5041668096,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 5078654976,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.34.ffn_up.weight",
"offset": 5115641856,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 5152628736,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 5152645120,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.34.attn_output.weight",
"offset": 5154365440,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.34.attn_q.weight",
"offset": 5168128000,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.34.attn_v.weight",
"offset": 5181890560,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.35.attn_norm.weight",
"offset": 5183610880,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 5183627264,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 5220614144,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.35.ffn_up.weight",
"offset": 5257601024,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 5294587904,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 5294604288,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.35.attn_output.weight",
"offset": 5296324608,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.35.attn_q.weight",
"offset": 5310087168,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.35.attn_v.weight",
"offset": 5323849728,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 5325570048,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 5325586432,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 5362573312,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.36.ffn_up.weight",
"offset": 5399560192,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 5436547072,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 5436563456,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.36.attn_output.weight",
"offset": 5438283776,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.36.attn_q.weight",
"offset": 5452046336,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.36.attn_v.weight",
"offset": 5465808896,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 5467529216,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 5467545600,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 5504532480,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.37.ffn_up.weight",
"offset": 5541519360,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 5578506240,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 5578522624,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.37.attn_output.weight",
"offset": 5580242944,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.37.attn_q.weight",
"offset": 5594005504,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.37.attn_v.weight",
"offset": 5607768064,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.38.attn_norm.weight",
"offset": 5609488384,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 5609504768,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 5646491648,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.38.ffn_up.weight",
"offset": 5683478528,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 5720465408,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 5720481792,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.38.attn_output.weight",
"offset": 5722202112,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.38.attn_q.weight",
"offset": 5735964672,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.38.attn_v.weight",
"offset": 5749727232,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 5751447552,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 5751463936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 5788450816,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.39.ffn_up.weight",
"offset": 5825437696,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 5862424576,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 5862440960,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.39.attn_output.weight",
"offset": 5864161280,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.39.attn_q.weight",
"offset": 5877923840,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.39.attn_v.weight",
"offset": 5891686400,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 5893406720,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 5893423104,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 5930409984,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.40.ffn_up.weight",
"offset": 5967396864,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 6004383744,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 6004400128,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.40.attn_output.weight",
"offset": 6006120448,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.40.attn_q.weight",
"offset": 6019883008,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.40.attn_v.weight",
"offset": 6033645568,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 6035365888,
"shape": [
4096,
11008
],
"size": 1720320,
"type": 14
},
{
"name": "blk.41.attn_k.weight",
"offset": 6072352768,
"shape": [
4096,
512
],
"size": 36986880,
"type": 14
},
{
"name": "blk.41.attn_output.weight",
"offset": 6074073088,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.41.attn_q.weight",
"offset": 6087835648,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.41.attn_v.weight",
"offset": 6101598208,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "output.weight",
"offset": 6103318528,
"shape": [
4096,
64000
],
"size": 1720320,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 6318358528,
"shape": [
4096
],
"size": 215040000,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 6318374912,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 6355361792,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 6392348672,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.42.attn_norm.weight",
"offset": 6392365056,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 6392381440,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 6429368320,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 6466355200,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 6503342080,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 6503358464,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.42.attn_output.weight",
"offset": 6505078784,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.42.attn_q.weight",
"offset": 6518841344,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.42.attn_v.weight",
"offset": 6532603904,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.43.attn_norm.weight",
"offset": 6534324224,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 6534340608,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 6571327488,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.43.ffn_up.weight",
"offset": 6608314368,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 6645301248,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 6645317632,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.43.attn_output.weight",
"offset": 6647037952,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.43.attn_q.weight",
"offset": 6660800512,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.43.attn_v.weight",
"offset": 6674563072,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 6676283392,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 6676299776,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 6713286656,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.44.ffn_up.weight",
"offset": 6750273536,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 6787260416,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 6787276800,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.44.attn_output.weight",
"offset": 6788997120,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.44.attn_q.weight",
"offset": 6802759680,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.44.attn_v.weight",
"offset": 6816522240,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 6818242560,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 6818258944,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 6855245824,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.45.ffn_up.weight",
"offset": 6892232704,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 6929219584,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 6929235968,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.45.attn_output.weight",
"offset": 6930956288,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.45.attn_q.weight",
"offset": 6944718848,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.45.attn_v.weight",
"offset": 6958481408,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 6960201728,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 6960218112,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 6997204992,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.46.ffn_up.weight",
"offset": 7034191872,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 7071178752,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 7071195136,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.46.attn_output.weight",
"offset": 7072915456,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.46.attn_q.weight",
"offset": 7086678016,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.46.attn_v.weight",
"offset": 7100440576,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "blk.47.attn_norm.weight",
"offset": 7102160896,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 7102177280,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 7139164160,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.47.ffn_up.weight",
"offset": 7176151040,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 14
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 7213137920,
"shape": [
4096
],
"size": 36986880,
"type": 0
},
{
"name": "\ufffd\u001f\ufffd-\ufffdJ\u0019\ufffd\ufffd\u0016\ufffd\ufffd\ufffd\ufffd\u0026\ufffde괥",
"offset": 7213154304,
"shape": [
4096,
512
],
"size": 16384,
"type": 14
},
{
"name": "blk.47.attn_output.weight",
"offset": 7214874624,
"shape": [
4096,
4096
],
"size": 1720320,
"type": 14
},
{
"name": "blk.47.attn_q.weight",
"offset": 7228637184,
"shape": [
4096,
4096
],
"size": 13762560,
"type": 14
},
{
"name": "blk.47.attn_v.weight",
"offset": 7242399744,
"shape": [
4096,
512
],
"size": 13762560,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 7244120064,
"shape": [
4096
],
"size": 1720320,
"type": 0
}
],
"version": 3
}