This model was created with the goal for a good llama 3 uncencored model with long context. At it worked like a charm.
70B
1,117 Pulls Updated 5 weeks ago
6d1c7eba2686 · 75GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 7,
"general.name": "Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/ryzen88/Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.url": "https://huggingface.co/mradermacher/Llama-3-70b-Uncensored-Lumi-Tess-gradient-GGUF",
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 262144,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 207112200,
"llama.vocab_size": 128256,
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-05-15T05:24:13+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "db1",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "token_embd.weight",
"offset": 1116340224,
"shape": [
8192,
128256
],
"size": 1116340224,
"type": 8
},
{
"name": "blk.0.attn_norm.weight",
"offset": 2232680448,
"shape": [
8192
],
"size": 1116340224,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 2232713216,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 2482274304,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 2731835392,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2981396480,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 2981429248,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 2990342144,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.0.attn_q.weight",
"offset": 3061645312,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.0.attn_v.weight",
"offset": 3132948480,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 3141861376,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 3141894144,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 3391455232,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 3641016320,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 3890577408,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 3890610176,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 3899523072,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.1.attn_q.weight",
"offset": 3970826240,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.1.attn_v.weight",
"offset": 4042129408,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 4051042304,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 4051075072,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 4300636160,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 4550197248,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 4799758336,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 4799791104,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 4808704000,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.10.attn_q.weight",
"offset": 4880007168,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.10.attn_v.weight",
"offset": 4951310336,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 4960223232,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 4960256000,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 5209817088,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 5459378176,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 5708939264,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 5708972032,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 5717884928,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.11.attn_q.weight",
"offset": 5789188096,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.11.attn_v.weight",
"offset": 5860491264,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 5869404160,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 5869436928,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 6118998016,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 6368559104,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 6618120192,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 6618152960,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 6627065856,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.12.attn_q.weight",
"offset": 6698369024,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.12.attn_v.weight",
"offset": 6769672192,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 6778585088,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 6778617856,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 7028178944,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 7277740032,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 7527301120,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 7527333888,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 7536246784,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.13.attn_q.weight",
"offset": 7607549952,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.13.attn_v.weight",
"offset": 7678853120,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 7687766016,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 7687798784,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 7937359872,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 8186920960,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 8436482048,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 8436514816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 8445427712,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.14.attn_q.weight",
"offset": 8516730880,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.14.attn_v.weight",
"offset": 8588034048,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 8596946944,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 8596979712,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 8846540800,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 9096101888,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 9345662976,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 9345695744,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 9354608640,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.15.attn_q.weight",
"offset": 9425911808,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.15.attn_v.weight",
"offset": 9497214976,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 9506127872,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 9506160640,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 9755721728,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 10005282816,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 10254843904,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 10254876672,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 10263789568,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.16.attn_q.weight",
"offset": 10335092736,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.16.attn_v.weight",
"offset": 10406395904,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 10415308800,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 10415341568,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 10664902656,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 10914463744,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 11164024832,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 11164057600,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 11172970496,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.17.attn_q.weight",
"offset": 11244273664,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.17.attn_v.weight",
"offset": 11315576832,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 11324489728,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 11324522496,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 11574083584,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 11823644672,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 12073205760,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 12073238528,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 12082151424,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.18.attn_q.weight",
"offset": 12153454592,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.18.attn_v.weight",
"offset": 12224757760,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 12233670656,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 12233703424,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 12483264512,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 12732825600,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 12982386688,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 12982419456,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 12991332352,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.19.attn_q.weight",
"offset": 13062635520,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.19.attn_v.weight",
"offset": 13133938688,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 13142851584,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 13142884352,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 13392445440,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 13642006528,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 13891567616,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 13891600384,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 13900513280,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.2.attn_q.weight",
"offset": 13971816448,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.2.attn_v.weight",
"offset": 14043119616,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 14052032512,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 14052065280,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 14301626368,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 14551187456,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 14800748544,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 14800781312,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 14809694208,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.20.attn_q.weight",
"offset": 14880997376,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.20.attn_v.weight",
"offset": 14952300544,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 14961213440,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 14961246208,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 15210807296,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 15460368384,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 15709929472,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 15709962240,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 15718875136,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.21.attn_q.weight",
"offset": 15790178304,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.21.attn_v.weight",
"offset": 15861481472,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 15870394368,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 15870427136,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 16119988224,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 16369549312,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 16619110400,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 16619143168,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 16628056064,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.22.attn_q.weight",
"offset": 16699359232,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.22.attn_v.weight",
"offset": 16770662400,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 16779575296,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 16779608064,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 17029169152,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 17278730240,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 17528291328,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 17528324096,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 17537236992,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.23.attn_q.weight",
"offset": 17608540160,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.23.attn_v.weight",
"offset": 17679843328,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 17688756224,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 17688788992,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 17938350080,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 18187911168,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 18437472256,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 18437505024,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 18446417920,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.24.attn_q.weight",
"offset": 18517721088,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.24.attn_v.weight",
"offset": 18589024256,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 18597937152,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 18597969920,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 18847531008,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 19097092096,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 19346653184,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 19346685952,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 19355598848,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.25.attn_q.weight",
"offset": 19426902016,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.25.attn_v.weight",
"offset": 19498205184,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 19507118080,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 19507150848,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 19756711936,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 20006273024,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 20255834112,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 20255866880,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 20264779776,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.26.attn_q.weight",
"offset": 20336082944,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.26.attn_v.weight",
"offset": 20407386112,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 20416299008,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 20416331776,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 20665892864,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 20915453952,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 21165015040,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 21165047808,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 21173960704,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.27.attn_q.weight",
"offset": 21245263872,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.27.attn_v.weight",
"offset": 21316567040,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 21325479936,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 21325512704,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 21575073792,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 21824634880,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 22074195968,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 22074228736,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 22083141632,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.28.attn_q.weight",
"offset": 22154444800,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.28.attn_v.weight",
"offset": 22225747968,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.29.attn_norm.weight",
"offset": 22234660864,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 22234693632,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 22484254720,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 22733815808,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 22983376896,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 22983409664,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.29.attn_output.weight",
"offset": 22992322560,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.29.attn_q.weight",
"offset": 23063625728,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.29.attn_v.weight",
"offset": 23134928896,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 23143841792,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 23143874560,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 23393435648,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 23642996736,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 23892557824,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 23892590592,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 23901503488,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.3.attn_q.weight",
"offset": 23972806656,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.3.attn_v.weight",
"offset": 24044109824,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 24053022720,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 24053055488,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 24302616576,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 24552177664,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 24801738752,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 24801771520,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.30.attn_output.weight",
"offset": 24810684416,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.30.attn_q.weight",
"offset": 24881987584,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.30.attn_v.weight",
"offset": 24953290752,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 24962203648,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 24962236416,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 25211797504,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 25461358592,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 25710919680,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 25710952448,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.31.attn_output.weight",
"offset": 25719865344,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.31.attn_q.weight",
"offset": 25791168512,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.31.attn_v.weight",
"offset": 25862471680,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.32.attn_norm.weight",
"offset": 25871384576,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 25871417344,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 26120978432,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.32.ffn_up.weight",
"offset": 26370539520,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 26620100608,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 26620133376,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.32.attn_output.weight",
"offset": 26629046272,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.32.attn_q.weight",
"offset": 26700349440,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.32.attn_v.weight",
"offset": 26771652608,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.33.attn_norm.weight",
"offset": 26780565504,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 26780598272,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 27030159360,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.33.ffn_up.weight",
"offset": 27279720448,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 27529281536,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 27529314304,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.33.attn_output.weight",
"offset": 27538227200,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.33.attn_q.weight",
"offset": 27609530368,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.33.attn_v.weight",
"offset": 27680833536,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.34.attn_norm.weight",
"offset": 27689746432,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 27689779200,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 27939340288,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.34.ffn_up.weight",
"offset": 28188901376,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 28438462464,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 28438495232,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.34.attn_output.weight",
"offset": 28447408128,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.34.attn_q.weight",
"offset": 28518711296,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.34.attn_v.weight",
"offset": 28590014464,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.35.attn_norm.weight",
"offset": 28598927360,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 28598960128,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 28848521216,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.35.ffn_up.weight",
"offset": 29098082304,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 29347643392,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 29347676160,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.35.attn_output.weight",
"offset": 29356589056,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.35.attn_q.weight",
"offset": 29427892224,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.35.attn_v.weight",
"offset": 29499195392,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.36.attn_norm.weight",
"offset": 29508108288,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 29508141056,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 29757702144,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.36.ffn_up.weight",
"offset": 30007263232,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 30256824320,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 30256857088,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.36.attn_output.weight",
"offset": 30265769984,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.36.attn_q.weight",
"offset": 30337073152,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.36.attn_v.weight",
"offset": 30408376320,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.37.attn_norm.weight",
"offset": 30417289216,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 30417321984,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 30666883072,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.37.ffn_up.weight",
"offset": 30916444160,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 31166005248,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 31166038016,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.37.attn_output.weight",
"offset": 31174950912,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.37.attn_q.weight",
"offset": 31246254080,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.37.attn_v.weight",
"offset": 31317557248,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.38.attn_norm.weight",
"offset": 31326470144,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 31326502912,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 31576064000,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.38.ffn_up.weight",
"offset": 31825625088,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 32075186176,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 32075218944,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.38.attn_output.weight",
"offset": 32084131840,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.38.attn_q.weight",
"offset": 32155435008,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.38.attn_v.weight",
"offset": 32226738176,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.39.attn_norm.weight",
"offset": 32235651072,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 32235683840,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 32485244928,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.39.ffn_up.weight",
"offset": 32734806016,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 32984367104,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 32984399872,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.39.attn_output.weight",
"offset": 32993312768,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.39.attn_q.weight",
"offset": 33064615936,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.39.attn_v.weight",
"offset": 33135919104,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 33144832000,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 33144864768,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 33394425856,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 33643986944,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 33893548032,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 33893580800,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 33902493696,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.4.attn_q.weight",
"offset": 33973796864,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.4.attn_v.weight",
"offset": 34045100032,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.40.attn_norm.weight",
"offset": 34054012928,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 34054045696,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 34303606784,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.40.ffn_up.weight",
"offset": 34553167872,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 34802728960,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 34802761728,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.40.attn_output.weight",
"offset": 34811674624,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.40.attn_q.weight",
"offset": 34882977792,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.40.attn_v.weight",
"offset": 34954280960,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.41.attn_norm.weight",
"offset": 34963193856,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 34963226624,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 35212787712,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.41.ffn_up.weight",
"offset": 35462348800,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 35711909888,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 35711942656,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.41.attn_output.weight",
"offset": 35720855552,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.41.attn_q.weight",
"offset": 35792158720,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.41.attn_v.weight",
"offset": 35863461888,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.42.attn_norm.weight",
"offset": 35872374784,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 35872407552,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 36121968640,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.42.ffn_up.weight",
"offset": 36371529728,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 36621090816,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 36621123584,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.42.attn_output.weight",
"offset": 36630036480,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.42.attn_q.weight",
"offset": 36701339648,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.42.attn_v.weight",
"offset": 36772642816,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.43.attn_norm.weight",
"offset": 36781555712,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 36781588480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 37031149568,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.43.ffn_up.weight",
"offset": 37280710656,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 37530271744,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 37530304512,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.43.attn_output.weight",
"offset": 37539217408,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.43.attn_q.weight",
"offset": 37610520576,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.43.attn_v.weight",
"offset": 37681823744,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.44.attn_norm.weight",
"offset": 37690736640,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 37690769408,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 37940330496,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.44.ffn_up.weight",
"offset": 38189891584,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 38439452672,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 38439485440,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.44.attn_output.weight",
"offset": 38448398336,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.44.attn_q.weight",
"offset": 38519701504,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.44.attn_v.weight",
"offset": 38591004672,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.45.attn_norm.weight",
"offset": 38599917568,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 38599950336,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 38849511424,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.45.ffn_up.weight",
"offset": 39099072512,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 39348633600,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 39348666368,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.45.attn_output.weight",
"offset": 39357579264,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.45.attn_q.weight",
"offset": 39428882432,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.45.attn_v.weight",
"offset": 39500185600,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.46.attn_norm.weight",
"offset": 39509098496,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 39509131264,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 39758692352,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.46.ffn_up.weight",
"offset": 40008253440,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 40257814528,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 40257847296,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.46.attn_output.weight",
"offset": 40266760192,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.46.attn_q.weight",
"offset": 40338063360,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.46.attn_v.weight",
"offset": 40409366528,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.47.attn_norm.weight",
"offset": 40418279424,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 40418312192,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 40667873280,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.47.ffn_up.weight",
"offset": 40917434368,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 41166995456,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 41167028224,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.47.attn_output.weight",
"offset": 41175941120,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.47.attn_q.weight",
"offset": 41247244288,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.47.attn_v.weight",
"offset": 41318547456,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.48.attn_norm.weight",
"offset": 41327460352,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 41327493120,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 41577054208,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.48.ffn_up.weight",
"offset": 41826615296,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 42076176384,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 42076209152,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.48.attn_output.weight",
"offset": 42085122048,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.48.attn_q.weight",
"offset": 42156425216,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.48.attn_v.weight",
"offset": 42227728384,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.49.attn_norm.weight",
"offset": 42236641280,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 42236674048,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 42486235136,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.49.ffn_up.weight",
"offset": 42735796224,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 42985357312,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 42985390080,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.49.attn_output.weight",
"offset": 42994302976,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.49.attn_q.weight",
"offset": 43065606144,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.49.attn_v.weight",
"offset": 43136909312,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 43145822208,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 43145854976,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 43395416064,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 43644977152,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 43894538240,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 43894571008,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 43903483904,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.5.attn_q.weight",
"offset": 43974787072,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.5.attn_v.weight",
"offset": 44046090240,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.50.attn_norm.weight",
"offset": 44055003136,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 44055035904,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 44304596992,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.50.ffn_up.weight",
"offset": 44554158080,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 44803719168,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 44803751936,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.50.attn_output.weight",
"offset": 44812664832,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.50.attn_q.weight",
"offset": 44883968000,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.50.attn_v.weight",
"offset": 44955271168,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.51.attn_norm.weight",
"offset": 44964184064,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 44964216832,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 45213777920,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.51.ffn_up.weight",
"offset": 45463339008,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 45712900096,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.51.attn_k.weight",
"offset": 45712932864,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.51.attn_output.weight",
"offset": 45721845760,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.51.attn_q.weight",
"offset": 45793148928,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.51.attn_v.weight",
"offset": 45864452096,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.52.attn_norm.weight",
"offset": 45873364992,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 45873397760,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 46122958848,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.52.ffn_up.weight",
"offset": 46372519936,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 46622081024,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 46622113792,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.52.attn_output.weight",
"offset": 46631026688,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.52.attn_q.weight",
"offset": 46702329856,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.52.attn_v.weight",
"offset": 46773633024,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.53.attn_norm.weight",
"offset": 46782545920,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 46782578688,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 47032139776,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.53.ffn_up.weight",
"offset": 47281700864,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 47531261952,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 47531294720,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.53.attn_output.weight",
"offset": 47540207616,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.53.attn_q.weight",
"offset": 47611510784,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.53.attn_v.weight",
"offset": 47682813952,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.54.attn_norm.weight",
"offset": 47691726848,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 47691759616,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 47941320704,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.54.ffn_up.weight",
"offset": 48190881792,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 48440442880,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 48440475648,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.54.attn_output.weight",
"offset": 48449388544,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.54.attn_q.weight",
"offset": 48520691712,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.54.attn_v.weight",
"offset": 48591994880,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.55.attn_norm.weight",
"offset": 48600907776,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 48600940544,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 48850501632,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.55.ffn_up.weight",
"offset": 49100062720,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 49349623808,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.55.attn_k.weight",
"offset": 49349656576,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.55.attn_output.weight",
"offset": 49358569472,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.55.attn_q.weight",
"offset": 49429872640,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.55.attn_v.weight",
"offset": 49501175808,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.56.attn_norm.weight",
"offset": 49510088704,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 49510121472,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 49759682560,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.56.ffn_up.weight",
"offset": 50009243648,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 50258804736,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 50258837504,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.56.attn_output.weight",
"offset": 50267750400,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.56.attn_q.weight",
"offset": 50339053568,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.56.attn_v.weight",
"offset": 50410356736,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.57.attn_norm.weight",
"offset": 50419269632,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 50419302400,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 50668863488,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.57.ffn_up.weight",
"offset": 50918424576,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 51167985664,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 51168018432,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.57.attn_output.weight",
"offset": 51176931328,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.57.attn_q.weight",
"offset": 51248234496,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.57.attn_v.weight",
"offset": 51319537664,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.58.attn_norm.weight",
"offset": 51328450560,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 51328483328,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 51578044416,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.58.ffn_up.weight",
"offset": 51827605504,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 52077166592,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 52077199360,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.58.attn_output.weight",
"offset": 52086112256,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.58.attn_q.weight",
"offset": 52157415424,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.58.attn_v.weight",
"offset": 52228718592,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.59.attn_norm.weight",
"offset": 52237631488,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 52237664256,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 52487225344,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.59.ffn_up.weight",
"offset": 52736786432,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 52986347520,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.59.attn_k.weight",
"offset": 52986380288,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.59.attn_output.weight",
"offset": 52995293184,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.59.attn_q.weight",
"offset": 53066596352,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.59.attn_v.weight",
"offset": 53137899520,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 53146812416,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 53146845184,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 53396406272,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 53645967360,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 53895528448,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 53895561216,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 53904474112,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.6.attn_q.weight",
"offset": 53975777280,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.6.attn_v.weight",
"offset": 54047080448,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.60.attn_norm.weight",
"offset": 54055993344,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.60.ffn_down.weight",
"offset": 54056026112,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 54305587200,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.60.ffn_up.weight",
"offset": 54555148288,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 54804709376,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.60.attn_k.weight",
"offset": 54804742144,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.60.attn_output.weight",
"offset": 54813655040,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.60.attn_q.weight",
"offset": 54884958208,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.60.attn_v.weight",
"offset": 54956261376,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.61.attn_norm.weight",
"offset": 54965174272,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.61.ffn_down.weight",
"offset": 54965207040,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 55214768128,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.61.ffn_up.weight",
"offset": 55464329216,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 55713890304,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.61.attn_k.weight",
"offset": 55713923072,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.61.attn_output.weight",
"offset": 55722835968,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.61.attn_q.weight",
"offset": 55794139136,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.61.attn_v.weight",
"offset": 55865442304,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.62.attn_norm.weight",
"offset": 55874355200,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.62.ffn_down.weight",
"offset": 55874387968,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 56123949056,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.62.ffn_up.weight",
"offset": 56373510144,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 56623071232,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.62.attn_k.weight",
"offset": 56623104000,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.62.attn_output.weight",
"offset": 56632016896,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.62.attn_q.weight",
"offset": 56703320064,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.62.attn_v.weight",
"offset": 56774623232,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.63.attn_norm.weight",
"offset": 56783536128,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.63.ffn_down.weight",
"offset": 56783568896,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 57033129984,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.63.ffn_up.weight",
"offset": 57282691072,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 57532252160,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.63.attn_k.weight",
"offset": 57532284928,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.63.attn_output.weight",
"offset": 57541197824,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.63.attn_q.weight",
"offset": 57612500992,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.63.attn_v.weight",
"offset": 57683804160,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.64.attn_norm.weight",
"offset": 57692717056,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.64.ffn_down.weight",
"offset": 57692749824,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 57942310912,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.64.ffn_up.weight",
"offset": 58191872000,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 58441433088,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.64.attn_k.weight",
"offset": 58441465856,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.64.attn_output.weight",
"offset": 58450378752,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.64.attn_q.weight",
"offset": 58521681920,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.64.attn_v.weight",
"offset": 58592985088,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.65.attn_norm.weight",
"offset": 58601897984,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.65.ffn_down.weight",
"offset": 58601930752,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 58851491840,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.65.ffn_up.weight",
"offset": 59101052928,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 59350614016,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.65.attn_k.weight",
"offset": 59350646784,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.65.attn_output.weight",
"offset": 59359559680,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.65.attn_q.weight",
"offset": 59430862848,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.65.attn_v.weight",
"offset": 59502166016,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.66.attn_norm.weight",
"offset": 59511078912,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.66.ffn_down.weight",
"offset": 59511111680,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 59760672768,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.66.ffn_up.weight",
"offset": 60010233856,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 60259794944,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.66.attn_k.weight",
"offset": 60259827712,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.66.attn_output.weight",
"offset": 60268740608,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.66.attn_q.weight",
"offset": 60340043776,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.66.attn_v.weight",
"offset": 60411346944,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.67.attn_norm.weight",
"offset": 60420259840,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.67.ffn_down.weight",
"offset": 60420292608,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 60669853696,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.67.ffn_up.weight",
"offset": 60919414784,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 61168975872,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.67.attn_k.weight",
"offset": 61169008640,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.67.attn_output.weight",
"offset": 61177921536,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.67.attn_q.weight",
"offset": 61249224704,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.67.attn_v.weight",
"offset": 61320527872,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.68.attn_norm.weight",
"offset": 61329440768,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.68.ffn_down.weight",
"offset": 61329473536,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 61579034624,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.68.ffn_up.weight",
"offset": 61828595712,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 62078156800,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.68.attn_k.weight",
"offset": 62078189568,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.68.attn_output.weight",
"offset": 62087102464,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.68.attn_q.weight",
"offset": 62158405632,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.68.attn_v.weight",
"offset": 62229708800,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.69.attn_norm.weight",
"offset": 62238621696,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.69.ffn_down.weight",
"offset": 62238654464,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 62488215552,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.69.ffn_up.weight",
"offset": 62737776640,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 62987337728,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.69.attn_k.weight",
"offset": 62987370496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.69.attn_output.weight",
"offset": 62996283392,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.69.attn_q.weight",
"offset": 63067586560,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.69.attn_v.weight",
"offset": 63138889728,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 63147802624,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 63147835392,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 63397396480,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 63646957568,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 63896518656,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 63896551424,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 63905464320,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.7.attn_q.weight",
"offset": 63976767488,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.7.attn_v.weight",
"offset": 64048070656,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.70.attn_norm.weight",
"offset": 64056983552,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.70.ffn_down.weight",
"offset": 64057016320,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 64306577408,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.70.ffn_up.weight",
"offset": 64556138496,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 64805699584,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.70.attn_k.weight",
"offset": 64805732352,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.70.attn_output.weight",
"offset": 64814645248,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.70.attn_q.weight",
"offset": 64885948416,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.70.attn_v.weight",
"offset": 64957251584,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.71.attn_norm.weight",
"offset": 64966164480,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.71.ffn_down.weight",
"offset": 64966197248,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 65215758336,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.71.ffn_up.weight",
"offset": 65465319424,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 65714880512,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.71.attn_k.weight",
"offset": 65714913280,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.71.attn_output.weight",
"offset": 65723826176,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.71.attn_q.weight",
"offset": 65795129344,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.71.attn_v.weight",
"offset": 65866432512,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.72.attn_norm.weight",
"offset": 65875345408,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.72.ffn_down.weight",
"offset": 65875378176,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 66124939264,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.72.ffn_up.weight",
"offset": 66374500352,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 66624061440,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.72.attn_k.weight",
"offset": 66624094208,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.72.attn_output.weight",
"offset": 66633007104,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.72.attn_q.weight",
"offset": 66704310272,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.72.attn_v.weight",
"offset": 66775613440,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.73.attn_norm.weight",
"offset": 66784526336,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.73.ffn_down.weight",
"offset": 66784559104,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 67034120192,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.73.ffn_up.weight",
"offset": 67283681280,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 67533242368,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.73.attn_k.weight",
"offset": 67533275136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.73.attn_output.weight",
"offset": 67542188032,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.73.attn_q.weight",
"offset": 67613491200,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.73.attn_v.weight",
"offset": 67684794368,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.74.attn_norm.weight",
"offset": 67693707264,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.74.ffn_down.weight",
"offset": 67693740032,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 67943301120,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.74.ffn_up.weight",
"offset": 68192862208,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 68442423296,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.74.attn_k.weight",
"offset": 68442456064,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.74.attn_output.weight",
"offset": 68451368960,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.74.attn_q.weight",
"offset": 68522672128,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.74.attn_v.weight",
"offset": 68593975296,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.75.attn_norm.weight",
"offset": 68602888192,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.75.ffn_down.weight",
"offset": 68602920960,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 68852482048,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.75.ffn_up.weight",
"offset": 69102043136,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 69351604224,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.75.attn_k.weight",
"offset": 69351636992,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.75.attn_output.weight",
"offset": 69360549888,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.75.attn_q.weight",
"offset": 69431853056,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.75.attn_v.weight",
"offset": 69503156224,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.76.attn_norm.weight",
"offset": 69512069120,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.76.ffn_down.weight",
"offset": 69512101888,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 69761662976,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.76.ffn_up.weight",
"offset": 70011224064,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 70260785152,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.76.attn_k.weight",
"offset": 70260817920,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.76.attn_output.weight",
"offset": 70269730816,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.76.attn_q.weight",
"offset": 70341033984,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.76.attn_v.weight",
"offset": 70412337152,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.77.attn_norm.weight",
"offset": 70421250048,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.77.ffn_down.weight",
"offset": 70421282816,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 70670843904,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.77.ffn_up.weight",
"offset": 70920404992,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 71169966080,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.77.attn_k.weight",
"offset": 71169998848,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.77.attn_output.weight",
"offset": 71178911744,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.77.attn_q.weight",
"offset": 71250214912,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.77.attn_v.weight",
"offset": 71321518080,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.78.attn_norm.weight",
"offset": 71330430976,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.78.ffn_down.weight",
"offset": 71330463744,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 71580024832,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.78.ffn_up.weight",
"offset": 71829585920,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 72079147008,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.78.attn_k.weight",
"offset": 72079179776,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.78.attn_output.weight",
"offset": 72088092672,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.78.attn_q.weight",
"offset": 72159395840,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.78.attn_v.weight",
"offset": 72230699008,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.79.attn_norm.weight",
"offset": 72239611904,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.79.ffn_down.weight",
"offset": 72239644672,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 72489205760,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.79.ffn_up.weight",
"offset": 72738766848,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 72988327936,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.79.attn_k.weight",
"offset": 72988360704,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.79.attn_output.weight",
"offset": 72997273600,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.79.attn_q.weight",
"offset": 73068576768,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.79.attn_v.weight",
"offset": 73139879936,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 73148792832,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 73148825600,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 73398386688,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 73647947776,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 73897508864,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 73897541632,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 73906454528,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.8.attn_q.weight",
"offset": 73977757696,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.8.attn_v.weight",
"offset": 74049060864,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 74057973760,
"shape": [
8192
],
"size": 8912896,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 74058006528,
"shape": [
28672,
8192
],
"size": 32768,
"type": 8
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 74307567616,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 74557128704,
"shape": [
8192,
28672
],
"size": 249561088,
"type": 8
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 74806689792,
"shape": [
8192
],
"size": 249561088,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 74806722560,
"shape": [
8192,
1024
],
"size": 32768,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 74815635456,
"shape": [
8192,
8192
],
"size": 8912896,
"type": 8
},
{
"name": "blk.9.attn_q.weight",
"offset": 74886938624,
"shape": [
8192,
8192
],
"size": 71303168,
"type": 8
},
{
"name": "blk.9.attn_v.weight",
"offset": 74958241792,
"shape": [
8192,
1024
],
"size": 71303168,
"type": 8
},
{
"name": "output_norm.weight",
"offset": 74967154688,
"shape": [
8192
],
"size": 8912896,
"type": 0
}
],
"version": 3
}