Advancing Open-source Large Language Models in Medical Domain , in Chinese or mutil-languages.
70B
270 Pulls Updated 5 weeks ago
cf7fb8dac551 · 75GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 7,
"general.name": ".",
"general.quantization_version": 2,
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 8192,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.scores": "... (128256 values)",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "blk.0.attn_q.weight",
"offset": 1116340224,
"shape": [
8192,
8192
],
"size": 1116340224,
"type": 8
},
{
"name": "blk.0.attn_k.weight",
"offset": 1187643392,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.0.attn_v.weight",
"offset": 1196556288,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 1205469184,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1276772352,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1526333440,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1775894528,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.0.attn_norm.weight",
"offset": 2025455616,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2025488384,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 2025521152,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.1.attn_k.weight",
"offset": 2096824320,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.1.attn_v.weight",
"offset": 2105737216,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 2114650112,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 2185953280,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 2435514368,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.1.ffn_down.weight",
"offset": 2685075456,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2934636544,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2934669312,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 2934702080,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.2.attn_k.weight",
"offset": 3006005248,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.2.attn_v.weight",
"offset": 3014918144,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 3023831040,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 3095134208,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 3344695296,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.2.ffn_down.weight",
"offset": 3594256384,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 3843817472,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 3843850240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 3843883008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.3.attn_k.weight",
"offset": 3915186176,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.3.attn_v.weight",
"offset": 3924099072,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 3933011968,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 4004315136,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 4253876224,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.3.ffn_down.weight",
"offset": 4503437312,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 4752998400,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 4753031168,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 4753063936,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.4.attn_k.weight",
"offset": 4824367104,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.4.attn_v.weight",
"offset": 4833280000,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 4842192896,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 4913496064,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 5163057152,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.4.ffn_down.weight",
"offset": 5412618240,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 5662179328,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 5662212096,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 5662244864,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.5.attn_k.weight",
"offset": 5733548032,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.5.attn_v.weight",
"offset": 5742460928,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 5751373824,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 5822676992,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 6072238080,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.5.ffn_down.weight",
"offset": 6321799168,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 6571360256,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 6571393024,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 6571425792,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.6.attn_k.weight",
"offset": 6642728960,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.6.attn_v.weight",
"offset": 6651641856,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 6660554752,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 6731857920,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 6981419008,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.6.ffn_down.weight",
"offset": 7230980096,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 7480541184,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 7480573952,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 7480606720,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.7.attn_k.weight",
"offset": 7551909888,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.7.attn_v.weight",
"offset": 7560822784,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 7569735680,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 7641038848,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 7890599936,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.7.ffn_down.weight",
"offset": 8140161024,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 8389722112,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 8389754880,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 8389787648,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.8.attn_k.weight",
"offset": 8461090816,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.8.attn_v.weight",
"offset": 8470003712,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 8478916608,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 8550219776,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 8799780864,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.8.ffn_down.weight",
"offset": 9049341952,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 9298903040,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 9298935808,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 9298968576,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.9.attn_k.weight",
"offset": 9370271744,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.9.attn_v.weight",
"offset": 9379184640,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 9388097536,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 9459400704,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 9708961792,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.9.ffn_down.weight",
"offset": 9958522880,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 10208083968,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 10208116736,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 10208149504,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.10.attn_k.weight",
"offset": 10279452672,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.10.attn_v.weight",
"offset": 10288365568,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 10297278464,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 10368581632,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 10618142720,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.10.ffn_down.weight",
"offset": 10867703808,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 11117264896,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 11117297664,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 11117330432,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.11.attn_k.weight",
"offset": 11188633600,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.11.attn_v.weight",
"offset": 11197546496,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 11206459392,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 11277762560,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 11527323648,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.11.ffn_down.weight",
"offset": 11776884736,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 12026445824,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 12026478592,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 12026511360,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.12.attn_k.weight",
"offset": 12097814528,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.12.attn_v.weight",
"offset": 12106727424,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 12115640320,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 12186943488,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 12436504576,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.12.ffn_down.weight",
"offset": 12686065664,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 12935626752,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 12935659520,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 12935692288,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.13.attn_k.weight",
"offset": 13006995456,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.13.attn_v.weight",
"offset": 13015908352,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 13024821248,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 13096124416,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 13345685504,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.13.ffn_down.weight",
"offset": 13595246592,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 13844807680,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 13844840448,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 13844873216,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.14.attn_k.weight",
"offset": 13916176384,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.14.attn_v.weight",
"offset": 13925089280,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 13934002176,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 14005305344,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 14254866432,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.14.ffn_down.weight",
"offset": 14504427520,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 14753988608,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 14754021376,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 14754054144,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.15.attn_k.weight",
"offset": 14825357312,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.15.attn_v.weight",
"offset": 14834270208,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 14843183104,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 14914486272,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 15164047360,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.15.ffn_down.weight",
"offset": 15413608448,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 15663169536,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 15663202304,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 15663235072,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.16.attn_k.weight",
"offset": 15734538240,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.16.attn_v.weight",
"offset": 15743451136,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 15752364032,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 15823667200,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 16073228288,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.16.ffn_down.weight",
"offset": 16322789376,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 16572350464,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 16572383232,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 16572416000,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.17.attn_k.weight",
"offset": 16643719168,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.17.attn_v.weight",
"offset": 16652632064,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 16661544960,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 16732848128,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 16982409216,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.17.ffn_down.weight",
"offset": 17231970304,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 17481531392,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 17481564160,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 17481596928,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.18.attn_k.weight",
"offset": 17552900096,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.18.attn_v.weight",
"offset": 17561812992,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 17570725888,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 17642029056,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 17891590144,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.18.ffn_down.weight",
"offset": 18141151232,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 18390712320,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 18390745088,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 18390777856,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.19.attn_k.weight",
"offset": 18462081024,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.19.attn_v.weight",
"offset": 18470993920,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 18479906816,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 18551209984,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 18800771072,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.19.ffn_down.weight",
"offset": 19050332160,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 19299893248,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 19299926016,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 19299958784,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.20.attn_k.weight",
"offset": 19371261952,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.20.attn_v.weight",
"offset": 19380174848,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 19389087744,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 19460390912,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 19709952000,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.20.ffn_down.weight",
"offset": 19959513088,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 20209074176,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 20209106944,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 20209139712,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.21.attn_k.weight",
"offset": 20280442880,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.21.attn_v.weight",
"offset": 20289355776,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 20298268672,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 20369571840,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 20619132928,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.21.ffn_down.weight",
"offset": 20868694016,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 21118255104,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 21118287872,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 21118320640,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.22.attn_k.weight",
"offset": 21189623808,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.22.attn_v.weight",
"offset": 21198536704,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 21207449600,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 21278752768,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 21528313856,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.22.ffn_down.weight",
"offset": 21777874944,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 22027436032,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 22027468800,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 22027501568,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.23.attn_k.weight",
"offset": 22098804736,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.23.attn_v.weight",
"offset": 22107717632,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 22116630528,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 22187933696,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 22437494784,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.23.ffn_down.weight",
"offset": 22687055872,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 22936616960,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 22936649728,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 22936682496,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.24.attn_k.weight",
"offset": 23007985664,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.24.attn_v.weight",
"offset": 23016898560,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 23025811456,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 23097114624,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 23346675712,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.24.ffn_down.weight",
"offset": 23596236800,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 23845797888,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 23845830656,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 23845863424,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.25.attn_k.weight",
"offset": 23917166592,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.25.attn_v.weight",
"offset": 23926079488,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 23934992384,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 24006295552,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 24255856640,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.25.ffn_down.weight",
"offset": 24505417728,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 24754978816,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 24755011584,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 24755044352,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.26.attn_k.weight",
"offset": 24826347520,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.26.attn_v.weight",
"offset": 24835260416,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 24844173312,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 24915476480,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 25165037568,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.26.ffn_down.weight",
"offset": 25414598656,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 25664159744,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 25664192512,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 25664225280,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.27.attn_k.weight",
"offset": 25735528448,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.27.attn_v.weight",
"offset": 25744441344,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 25753354240,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 25824657408,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 26074218496,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.27.ffn_down.weight",
"offset": 26323779584,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 26573340672,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 26573373440,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 26573406208,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.28.attn_k.weight",
"offset": 26644709376,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.28.attn_v.weight",
"offset": 26653622272,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 26662535168,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 26733838336,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 26983399424,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.28.ffn_down.weight",
"offset": 27232960512,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 27482521600,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 27482554368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 27482587136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.29.attn_k.weight",
"offset": 27553890304,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.29.attn_v.weight",
"offset": 27562803200,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 27571716096,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 27643019264,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 27892580352,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.29.ffn_down.weight",
"offset": 28142141440,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.29.attn_norm.weight",
"offset": 28391702528,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 28391735296,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 28391768064,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.30.attn_k.weight",
"offset": 28463071232,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.30.attn_v.weight",
"offset": 28471984128,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 28480897024,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 28552200192,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 28801761280,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.30.ffn_down.weight",
"offset": 29051322368,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 29300883456,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 29300916224,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 29300948992,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.31.attn_k.weight",
"offset": 29372252160,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.31.attn_v.weight",
"offset": 29381165056,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 29390077952,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 29461381120,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 29710942208,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.31.ffn_down.weight",
"offset": 29960503296,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 30210064384,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 30210097152,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.32.attn_q.weight",
"offset": 30210129920,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.32.attn_k.weight",
"offset": 30281433088,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.32.attn_v.weight",
"offset": 30290345984,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.32.attn_output.weight",
"offset": 30299258880,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 30370562048,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.32.ffn_up.weight",
"offset": 30620123136,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.32.ffn_down.weight",
"offset": 30869684224,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.32.attn_norm.weight",
"offset": 31119245312,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 31119278080,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.33.attn_q.weight",
"offset": 31119310848,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.33.attn_k.weight",
"offset": 31190614016,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.33.attn_v.weight",
"offset": 31199526912,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.33.attn_output.weight",
"offset": 31208439808,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 31279742976,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.33.ffn_up.weight",
"offset": 31529304064,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.33.ffn_down.weight",
"offset": 31778865152,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.33.attn_norm.weight",
"offset": 32028426240,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 32028459008,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.34.attn_q.weight",
"offset": 32028491776,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.34.attn_k.weight",
"offset": 32099794944,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.34.attn_v.weight",
"offset": 32108707840,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.34.attn_output.weight",
"offset": 32117620736,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 32188923904,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.34.ffn_up.weight",
"offset": 32438484992,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.34.ffn_down.weight",
"offset": 32688046080,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.34.attn_norm.weight",
"offset": 32937607168,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 32937639936,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.35.attn_q.weight",
"offset": 32937672704,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.35.attn_k.weight",
"offset": 33008975872,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.35.attn_v.weight",
"offset": 33017888768,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.35.attn_output.weight",
"offset": 33026801664,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 33098104832,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.35.ffn_up.weight",
"offset": 33347665920,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.35.ffn_down.weight",
"offset": 33597227008,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.35.attn_norm.weight",
"offset": 33846788096,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 33846820864,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.36.attn_q.weight",
"offset": 33846853632,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.36.attn_k.weight",
"offset": 33918156800,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.36.attn_v.weight",
"offset": 33927069696,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.36.attn_output.weight",
"offset": 33935982592,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 34007285760,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.36.ffn_up.weight",
"offset": 34256846848,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.36.ffn_down.weight",
"offset": 34506407936,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.36.attn_norm.weight",
"offset": 34755969024,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 34756001792,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.37.attn_q.weight",
"offset": 34756034560,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.37.attn_k.weight",
"offset": 34827337728,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.37.attn_v.weight",
"offset": 34836250624,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.37.attn_output.weight",
"offset": 34845163520,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 34916466688,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.37.ffn_up.weight",
"offset": 35166027776,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.37.ffn_down.weight",
"offset": 35415588864,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.37.attn_norm.weight",
"offset": 35665149952,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 35665182720,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.38.attn_q.weight",
"offset": 35665215488,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.38.attn_k.weight",
"offset": 35736518656,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.38.attn_v.weight",
"offset": 35745431552,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.38.attn_output.weight",
"offset": 35754344448,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 35825647616,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.38.ffn_up.weight",
"offset": 36075208704,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.38.ffn_down.weight",
"offset": 36324769792,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.38.attn_norm.weight",
"offset": 36574330880,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 36574363648,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.39.attn_q.weight",
"offset": 36574396416,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.39.attn_k.weight",
"offset": 36645699584,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.39.attn_v.weight",
"offset": 36654612480,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.39.attn_output.weight",
"offset": 36663525376,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 36734828544,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.39.ffn_up.weight",
"offset": 36984389632,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.39.ffn_down.weight",
"offset": 37233950720,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.39.attn_norm.weight",
"offset": 37483511808,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 37483544576,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.40.attn_q.weight",
"offset": 37483577344,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.40.attn_k.weight",
"offset": 37554880512,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.40.attn_v.weight",
"offset": 37563793408,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.40.attn_output.weight",
"offset": 37572706304,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 37644009472,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.40.ffn_up.weight",
"offset": 37893570560,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.40.ffn_down.weight",
"offset": 38143131648,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.40.attn_norm.weight",
"offset": 38392692736,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 38392725504,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.41.attn_q.weight",
"offset": 38392758272,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.41.attn_k.weight",
"offset": 38464061440,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.41.attn_v.weight",
"offset": 38472974336,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.41.attn_output.weight",
"offset": 38481887232,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 38553190400,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.41.ffn_up.weight",
"offset": 38802751488,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.41.ffn_down.weight",
"offset": 39052312576,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.41.attn_norm.weight",
"offset": 39301873664,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 39301906432,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.42.attn_q.weight",
"offset": 39301939200,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.42.attn_k.weight",
"offset": 39373242368,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.42.attn_v.weight",
"offset": 39382155264,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.42.attn_output.weight",
"offset": 39391068160,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 39462371328,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.42.ffn_up.weight",
"offset": 39711932416,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.42.ffn_down.weight",
"offset": 39961493504,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.42.attn_norm.weight",
"offset": 40211054592,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 40211087360,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.43.attn_q.weight",
"offset": 40211120128,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.43.attn_k.weight",
"offset": 40282423296,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.43.attn_v.weight",
"offset": 40291336192,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.43.attn_output.weight",
"offset": 40300249088,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 40371552256,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.43.ffn_up.weight",
"offset": 40621113344,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.43.ffn_down.weight",
"offset": 40870674432,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.43.attn_norm.weight",
"offset": 41120235520,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 41120268288,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.44.attn_q.weight",
"offset": 41120301056,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.44.attn_k.weight",
"offset": 41191604224,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.44.attn_v.weight",
"offset": 41200517120,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.44.attn_output.weight",
"offset": 41209430016,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 41280733184,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.44.ffn_up.weight",
"offset": 41530294272,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.44.ffn_down.weight",
"offset": 41779855360,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.44.attn_norm.weight",
"offset": 42029416448,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 42029449216,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.45.attn_q.weight",
"offset": 42029481984,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.45.attn_k.weight",
"offset": 42100785152,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.45.attn_v.weight",
"offset": 42109698048,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.45.attn_output.weight",
"offset": 42118610944,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 42189914112,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.45.ffn_up.weight",
"offset": 42439475200,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.45.ffn_down.weight",
"offset": 42689036288,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.45.attn_norm.weight",
"offset": 42938597376,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 42938630144,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.46.attn_q.weight",
"offset": 42938662912,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.46.attn_k.weight",
"offset": 43009966080,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.46.attn_v.weight",
"offset": 43018878976,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.46.attn_output.weight",
"offset": 43027791872,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 43099095040,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.46.ffn_up.weight",
"offset": 43348656128,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.46.ffn_down.weight",
"offset": 43598217216,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.46.attn_norm.weight",
"offset": 43847778304,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 43847811072,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.47.attn_q.weight",
"offset": 43847843840,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.47.attn_k.weight",
"offset": 43919147008,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.47.attn_v.weight",
"offset": 43928059904,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.47.attn_output.weight",
"offset": 43936972800,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 44008275968,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.47.ffn_up.weight",
"offset": 44257837056,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.47.ffn_down.weight",
"offset": 44507398144,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.47.attn_norm.weight",
"offset": 44756959232,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 44756992000,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.48.attn_q.weight",
"offset": 44757024768,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.48.attn_k.weight",
"offset": 44828327936,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.48.attn_v.weight",
"offset": 44837240832,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.48.attn_output.weight",
"offset": 44846153728,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 44917456896,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.48.ffn_up.weight",
"offset": 45167017984,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.48.ffn_down.weight",
"offset": 45416579072,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.48.attn_norm.weight",
"offset": 45666140160,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 45666172928,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.49.attn_q.weight",
"offset": 45666205696,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.49.attn_k.weight",
"offset": 45737508864,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.49.attn_v.weight",
"offset": 45746421760,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.49.attn_output.weight",
"offset": 45755334656,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 45826637824,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.49.ffn_up.weight",
"offset": 46076198912,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.49.ffn_down.weight",
"offset": 46325760000,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.49.attn_norm.weight",
"offset": 46575321088,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 46575353856,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.50.attn_q.weight",
"offset": 46575386624,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.50.attn_k.weight",
"offset": 46646689792,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.50.attn_v.weight",
"offset": 46655602688,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.50.attn_output.weight",
"offset": 46664515584,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 46735818752,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.50.ffn_up.weight",
"offset": 46985379840,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.50.ffn_down.weight",
"offset": 47234940928,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.50.attn_norm.weight",
"offset": 47484502016,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 47484534784,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.51.attn_q.weight",
"offset": 47484567552,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.51.attn_k.weight",
"offset": 47555870720,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.51.attn_v.weight",
"offset": 47564783616,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.51.attn_output.weight",
"offset": 47573696512,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 47644999680,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.51.ffn_up.weight",
"offset": 47894560768,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.51.ffn_down.weight",
"offset": 48144121856,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.51.attn_norm.weight",
"offset": 48393682944,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 48393715712,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.52.attn_q.weight",
"offset": 48393748480,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.52.attn_k.weight",
"offset": 48465051648,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.52.attn_v.weight",
"offset": 48473964544,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.52.attn_output.weight",
"offset": 48482877440,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 48554180608,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.52.ffn_up.weight",
"offset": 48803741696,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.52.ffn_down.weight",
"offset": 49053302784,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.52.attn_norm.weight",
"offset": 49302863872,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 49302896640,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.53.attn_q.weight",
"offset": 49302929408,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.53.attn_k.weight",
"offset": 49374232576,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.53.attn_v.weight",
"offset": 49383145472,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.53.attn_output.weight",
"offset": 49392058368,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 49463361536,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.53.ffn_up.weight",
"offset": 49712922624,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.53.ffn_down.weight",
"offset": 49962483712,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.53.attn_norm.weight",
"offset": 50212044800,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 50212077568,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.54.attn_q.weight",
"offset": 50212110336,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.54.attn_k.weight",
"offset": 50283413504,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.54.attn_v.weight",
"offset": 50292326400,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.54.attn_output.weight",
"offset": 50301239296,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 50372542464,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.54.ffn_up.weight",
"offset": 50622103552,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.54.ffn_down.weight",
"offset": 50871664640,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.54.attn_norm.weight",
"offset": 51121225728,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 51121258496,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.55.attn_q.weight",
"offset": 51121291264,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.55.attn_k.weight",
"offset": 51192594432,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.55.attn_v.weight",
"offset": 51201507328,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.55.attn_output.weight",
"offset": 51210420224,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 51281723392,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.55.ffn_up.weight",
"offset": 51531284480,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.55.ffn_down.weight",
"offset": 51780845568,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.55.attn_norm.weight",
"offset": 52030406656,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 52030439424,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.56.attn_q.weight",
"offset": 52030472192,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.56.attn_k.weight",
"offset": 52101775360,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.56.attn_v.weight",
"offset": 52110688256,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.56.attn_output.weight",
"offset": 52119601152,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 52190904320,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.56.ffn_up.weight",
"offset": 52440465408,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.56.ffn_down.weight",
"offset": 52690026496,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.56.attn_norm.weight",
"offset": 52939587584,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 52939620352,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.57.attn_q.weight",
"offset": 52939653120,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.57.attn_k.weight",
"offset": 53010956288,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.57.attn_v.weight",
"offset": 53019869184,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.57.attn_output.weight",
"offset": 53028782080,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 53100085248,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.57.ffn_up.weight",
"offset": 53349646336,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.57.ffn_down.weight",
"offset": 53599207424,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.57.attn_norm.weight",
"offset": 53848768512,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 53848801280,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.58.attn_q.weight",
"offset": 53848834048,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.58.attn_k.weight",
"offset": 53920137216,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.58.attn_v.weight",
"offset": 53929050112,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.58.attn_output.weight",
"offset": 53937963008,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 54009266176,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.58.ffn_up.weight",
"offset": 54258827264,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.58.ffn_down.weight",
"offset": 54508388352,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.58.attn_norm.weight",
"offset": 54757949440,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 54757982208,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.59.attn_q.weight",
"offset": 54758014976,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.59.attn_k.weight",
"offset": 54829318144,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.59.attn_v.weight",
"offset": 54838231040,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.59.attn_output.weight",
"offset": 54847143936,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 54918447104,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.59.ffn_up.weight",
"offset": 55168008192,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.59.ffn_down.weight",
"offset": 55417569280,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.59.attn_norm.weight",
"offset": 55667130368,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 55667163136,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.60.attn_q.weight",
"offset": 55667195904,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "\b\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\f\u0000\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000",
"offset": 55738499072,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.60.attn_v.weight",
"offset": 55747411968,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.60.attn_output.weight",
"offset": 55756324864,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 55827628032,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.60.ffn_up.weight",
"offset": 56077189120,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.60.ffn_down.weight",
"offset": 56326750208,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.60.attn_norm.weight",
"offset": 56576311296,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 56576344064,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.61.attn_q.weight",
"offset": 56576376832,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.61.attn_k.weight",
"offset": 56647680000,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.61.attn_v.weight",
"offset": 56656592896,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.61.attn_output.weight",
"offset": 56665505792,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 56736808960,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.61.ffn_up.weight",
"offset": 56986370048,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.61.ffn_down.weight",
"offset": 57235931136,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.61.attn_norm.weight",
"offset": 57485492224,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 57485524992,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.62.attn_q.weight",
"offset": 57485557760,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.62.attn_k.weight",
"offset": 57556860928,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.62.attn_v.weight",
"offset": 57565773824,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.62.attn_output.weight",
"offset": 57574686720,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 57645989888,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.62.ffn_up.weight",
"offset": 57895550976,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.62.ffn_down.weight",
"offset": 58145112064,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.62.attn_norm.weight",
"offset": 58394673152,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 58394705920,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.63.attn_q.weight",
"offset": 58394738688,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.63.attn_k.weight",
"offset": 58466041856,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.63.attn_v.weight",
"offset": 58474954752,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.63.attn_output.weight",
"offset": 58483867648,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 58555170816,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.63.ffn_up.weight",
"offset": 58804731904,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.63.ffn_down.weight",
"offset": 59054292992,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.63.attn_norm.weight",
"offset": 59303854080,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 59303886848,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.64.attn_q.weight",
"offset": 59303919616,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.64.attn_k.weight",
"offset": 59375222784,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.64.attn_v.weight",
"offset": 59384135680,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.64.attn_output.weight",
"offset": 59393048576,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 59464351744,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.64.ffn_up.weight",
"offset": 59713912832,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.64.ffn_down.weight",
"offset": 59963473920,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.64.attn_norm.weight",
"offset": 60213035008,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 60213067776,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.65.attn_q.weight",
"offset": 60213100544,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.65.attn_k.weight",
"offset": 60284403712,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.65.attn_v.weight",
"offset": 60293316608,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.65.attn_output.weight",
"offset": 60302229504,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 60373532672,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.65.ffn_up.weight",
"offset": 60623093760,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.65.ffn_down.weight",
"offset": 60872654848,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.65.attn_norm.weight",
"offset": 61122215936,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 61122248704,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.66.attn_q.weight",
"offset": 61122281472,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.66.attn_k.weight",
"offset": 61193584640,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.66.attn_v.weight",
"offset": 61202497536,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.66.attn_output.weight",
"offset": 61211410432,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 61282713600,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.66.ffn_up.weight",
"offset": 61532274688,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.66.ffn_down.weight",
"offset": 61781835776,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.66.attn_norm.weight",
"offset": 62031396864,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 62031429632,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.67.attn_q.weight",
"offset": 62031462400,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.67.attn_k.weight",
"offset": 62102765568,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.67.attn_v.weight",
"offset": 62111678464,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.67.attn_output.weight",
"offset": 62120591360,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 62191894528,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.67.ffn_up.weight",
"offset": 62441455616,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.67.ffn_down.weight",
"offset": 62691016704,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.67.attn_norm.weight",
"offset": 62940577792,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 62940610560,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.68.attn_q.weight",
"offset": 62940643328,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.68.attn_k.weight",
"offset": 63011946496,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.68.attn_v.weight",
"offset": 63020859392,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.68.attn_output.weight",
"offset": 63029772288,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 63101075456,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.68.ffn_up.weight",
"offset": 63350636544,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.68.ffn_down.weight",
"offset": 63600197632,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.68.attn_norm.weight",
"offset": 63849758720,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 63849791488,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.69.attn_q.weight",
"offset": 63849824256,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.69.attn_k.weight",
"offset": 63921127424,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.69.attn_v.weight",
"offset": 63930040320,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.69.attn_output.weight",
"offset": 63938953216,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 64010256384,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.69.ffn_up.weight",
"offset": 64259817472,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.69.ffn_down.weight",
"offset": 64509378560,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.69.attn_norm.weight",
"offset": 64758939648,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 64758972416,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.70.attn_q.weight",
"offset": 64759005184,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.70.attn_k.weight",
"offset": 64830308352,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.70.attn_v.weight",
"offset": 64839221248,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.70.attn_output.weight",
"offset": 64848134144,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 64919437312,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.70.ffn_up.weight",
"offset": 65168998400,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.70.ffn_down.weight",
"offset": 65418559488,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.70.attn_norm.weight",
"offset": 65668120576,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 65668153344,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.71.attn_q.weight",
"offset": 65668186112,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.71.attn_k.weight",
"offset": 65739489280,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.71.attn_v.weight",
"offset": 65748402176,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.71.attn_output.weight",
"offset": 65757315072,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 65828618240,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.71.ffn_up.weight",
"offset": 66078179328,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.71.ffn_down.weight",
"offset": 66327740416,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.71.attn_norm.weight",
"offset": 66577301504,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 66577334272,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.72.attn_q.weight",
"offset": 66577367040,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.72.attn_k.weight",
"offset": 66648670208,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.72.attn_v.weight",
"offset": 66657583104,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.72.attn_output.weight",
"offset": 66666496000,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 66737799168,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.72.ffn_up.weight",
"offset": 66987360256,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.72.ffn_down.weight",
"offset": 67236921344,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.72.attn_norm.weight",
"offset": 67486482432,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 67486515200,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.73.attn_q.weight",
"offset": 67486547968,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.73.attn_k.weight",
"offset": 67557851136,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.73.attn_v.weight",
"offset": 67566764032,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.73.attn_output.weight",
"offset": 67575676928,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 67646980096,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.73.ffn_up.weight",
"offset": 67896541184,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.73.ffn_down.weight",
"offset": 68146102272,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.73.attn_norm.weight",
"offset": 68395663360,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 68395696128,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.74.attn_q.weight",
"offset": 68395728896,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.74.attn_k.weight",
"offset": 68467032064,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.74.attn_v.weight",
"offset": 68475944960,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.74.attn_output.weight",
"offset": 68484857856,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 68556161024,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.74.ffn_up.weight",
"offset": 68805722112,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.74.ffn_down.weight",
"offset": 69055283200,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.74.attn_norm.weight",
"offset": 69304844288,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 69304877056,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.75.attn_q.weight",
"offset": 69304909824,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.75.attn_k.weight",
"offset": 69376212992,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.75.attn_v.weight",
"offset": 69385125888,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "\ufffd\ufffd\ufffd\ufffd\b\ufffd\ufffd\ufffd\ufffdܽ\t\"\ufffd\ufffd\u0015\ufffd\ufffd\u0005\ufffd\ufffd\u001e4M\u0006",
"offset": 69394038784,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 69465341952,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.75.ffn_up.weight",
"offset": 69714903040,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.75.ffn_down.weight",
"offset": 69964464128,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.75.attn_norm.weight",
"offset": 70214025216,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 70214057984,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.76.attn_q.weight",
"offset": 70214090752,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.76.attn_k.weight",
"offset": 70285393920,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.76.attn_v.weight",
"offset": 70294306816,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.76.attn_output.weight",
"offset": 70303219712,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 70374522880,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.76.ffn_up.weight",
"offset": 70624083968,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.76.ffn_down.weight",
"offset": 70873645056,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.76.attn_norm.weight",
"offset": 71123206144,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 71123238912,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.77.attn_q.weight",
"offset": 71123271680,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.77.attn_k.weight",
"offset": 71194574848,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.77.attn_v.weight",
"offset": 71203487744,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.77.attn_output.weight",
"offset": 71212400640,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 71283703808,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.77.ffn_up.weight",
"offset": 71533264896,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.77.ffn_down.weight",
"offset": 71782825984,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.77.attn_norm.weight",
"offset": 72032387072,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 72032419840,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.78.attn_q.weight",
"offset": 72032452608,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.78.attn_k.weight",
"offset": 72103755776,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.78.attn_v.weight",
"offset": 72112668672,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.78.attn_output.weight",
"offset": 72121581568,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 72192884736,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.78.ffn_up.weight",
"offset": 72442445824,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.78.ffn_down.weight",
"offset": 72692006912,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.78.attn_norm.weight",
"offset": 72941568000,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 72941600768,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.79.attn_q.weight",
"offset": 72941633536,
"shape": [
8192,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.79.attn_k.weight",
"offset": 73012936704,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.79.attn_v.weight",
"offset": 73021849600,
"shape": [
8192,
1024
],
"size": 8912896,
"type": 8
},
{
"name": "blk.79.attn_output.weight",
"offset": 73030762496,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 73102065664,
"shape": [
8192,
28672
],
"size": 71303168,
"type": 8
},
{
"name": "blk.79.ffn_up.weight",
"offset": 73351626752,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.79.ffn_down.weight",
"offset": 73601187840,
"shape": [
28672,
8192
],
"size": 249561088,
"type": 8
},
{
"name": "blk.79.attn_norm.weight",
"offset": 73850748928,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 73850781696,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 73850814464,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output.weight",
"offset": 73850847232,
"shape": [
8192,
128256
],
"size": 32768,
"type": 8
}
],
"version": 3
}