yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 7, "general.name": "Yi-1.5-9B", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 278528000, "shape": [ 4096 ], "size": 278528000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 278544384, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 326451200, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 374358016, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 422264832, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 422281216, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 424509440, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 442335232, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.0.attn_v.weight", "offset": 460161024, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 462389248, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 462405632, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 510312448, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 558219264, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 606126080, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 606142464, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 608370688, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 626196480, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_v.weight", "offset": 644022272, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 646250496, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 646266880, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 694173696, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 742080512, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 789987328, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 790003712, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 792231936, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 810057728, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_v.weight", "offset": 827883520, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 830111744, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 830128128, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 878034944, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 925941760, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 973848576, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 973864960, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 976093184, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.11.attn_q.weight", "offset": 993918976, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_v.weight", "offset": 1011744768, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 1013972992, "shape": [ 4096, 11008 ], "size": 2228224, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 1061879808, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.12.attn_k.weight", "offset": 1109786624, "shape": [ 4096, 512 ], "size": 47906816, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 1112014848, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 1129840640, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_v.weight", "offset": 1147666432, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1149894656, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1149911040, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 1197817856, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 1245724672, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 1293631488, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1293647872, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1295876096, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 1313701888, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_v.weight", "offset": 1331527680, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1333755904, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1333772288, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1381679104, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1429585920, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 1477492736, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1477509120, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1479737344, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 1497563136, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_v.weight", "offset": 1515388928, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1517617152, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1517633536, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 1565540352, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1613447168, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 1661353984, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1661370368, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1663598592, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 1681424384, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_v.weight", "offset": 1699250176, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1701478400, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1701494784, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1749401600, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1797308416, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 1845215232, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1845231616, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 1847459840, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 1865285632, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_v.weight", "offset": 1883111424, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 1885339648, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1885356032, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 1933262848, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 1981169664, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 2029076480, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2029092864, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 2031321088, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 2049146880, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_v.weight", "offset": 2066972672, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 2069200896, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2069217280, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 2117124096, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 2165030912, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 2212937728, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2212954112, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 2215182336, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 2233008128, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_v.weight", "offset": 2250833920, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 2253062144, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2253078528, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 2300985344, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 2348892160, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 2396798976, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2396815360, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 2399043584, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 2416869376, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_v.weight", "offset": 2434695168, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 2436923392, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2436939776, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 2484846592, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 2532753408, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 2580660224, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2580676608, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 2582904832, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 2600730624, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_v.weight", "offset": 2618556416, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 2620784640, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2620801024, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 2668707840, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 2668724224, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2668740608, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 2716647424, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 2764554240, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 2812461056, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2812477440, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 2814705664, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 2832531456, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.13.attn_v.weight", "offset": 2850357248, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 2852585472, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2852601856, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 2900508672, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 2948415488, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 2996322304, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2996338688, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 2998566912, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 3016392704, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_v.weight", "offset": 3034218496, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 3036446720, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3036463104, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 3084369920, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 3132276736, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 3180183552, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3180199936, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 3182428160, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 3200253952, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.15.attn_v.weight", "offset": 3218079744, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 3220307968, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3220324352, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 3268231168, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 3316137984, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 3364044800, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3364061184, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 3366289408, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.16.attn_q.weight", "offset": 3384115200, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_v.weight", "offset": 3401940992, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 3404169216, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3404185600, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 3452092416, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 3499999232, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 3547906048, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3547922432, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 3550150656, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 3567976448, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_v.weight", "offset": 3585802240, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 3588030464, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3588046848, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 3635953664, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 3683860480, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 3731767296, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3731783680, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 3734011904, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.18.attn_q.weight", "offset": 3751837696, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_v.weight", "offset": 3769663488, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 3771891712, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3771908096, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 3819814912, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 3867721728, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 3915628544, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3915644928, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 3917873152, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 3935698944, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_v.weight", "offset": 3953524736, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 3955752960, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3955769344, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 4003676160, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 4051582976, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 4099489792, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4099506176, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 4101734400, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 4119560192, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_v.weight", "offset": 4137385984, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 4139614208, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4139630592, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 4187537408, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 4235444224, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 4283351040, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4283367424, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 4285595648, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 4303421440, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.21.attn_v.weight", "offset": 4321247232, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 4323475456, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4323491840, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 4371398656, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 4419305472, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 4467212288, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4467228672, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 4469456896, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 4487282688, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_v.weight", "offset": 4505108480, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 4507336704, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4507353088, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 4555259904, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 4603166720, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 4651073536, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4651089920, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 4653318144, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 4671143936, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_v.weight", "offset": 4688969728, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 4691197952, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4691214336, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 4739121152, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 4787027968, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 4834934784, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4834951168, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 4837179392, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 4855005184, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_v.weight", "offset": 4872830976, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 4875059200, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4875075584, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 4922982400, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 4970889216, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 5018796032, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5018812416, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 5021040640, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 5038866432, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_v.weight", "offset": 5056692224, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 5058920448, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5058936832, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 5106843648, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 5154750464, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 5202657280, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5202673664, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 5204901888, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 5222727680, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_v.weight", "offset": 5240553472, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_k.weight", "offset": 5242781696, "shape": [ 4096, 512 ], "size": 2228224, "type": 8 }, { "name": "blk.27.attn_q.weight", "offset": 5245009920, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.27.attn_v.weight", "offset": 5262835712, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 5265063936, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5265080320, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 5312987136, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 5360893952, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 5408800768, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5408817152, "shape": [ 4096, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 5426642944, "shape": [ 4096 ], "size": 17825792, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5426659328, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.28.ffn_gate.weight", "offset": 5474566144, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 5522472960, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 5570379776, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 5570396160, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 5572624384, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.28.attn_q.weight", "offset": 5590450176, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.28.attn_v.weight", "offset": 5608275968, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 5610504192, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5610520576, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.29.ffn_gate.weight", "offset": 5658427392, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 5706334208, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 5754241024, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 5754257408, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 5756485632, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.29.attn_q.weight", "offset": 5774311424, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_v.weight", "offset": 5792137216, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 5794365440, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5794381824, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.30.ffn_gate.weight", "offset": 5842288640, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 5890195456, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 5938102272, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 5938118656, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 5940346880, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.30.attn_q.weight", "offset": 5958172672, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_v.weight", "offset": 5975998464, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 5978226688, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5978243072, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.31.ffn_gate.weight", "offset": 6026149888, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 6074056704, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 6121963520, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 6121979904, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 6124208128, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.31.attn_q.weight", "offset": 6142033920, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.31.attn_v.weight", "offset": 6159859712, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 6162087936, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6162104320, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.32.ffn_gate.weight", "offset": 6210011136, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 6257917952, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.32.ffn_norm.weight", "offset": 6305824768, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 6305841152, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 6308069376, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.32.attn_q.weight", "offset": 6325895168, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.32.attn_v.weight", "offset": 6343720960, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.33.attn_norm.weight", "offset": 6345949184, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6345965568, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.33.ffn_gate.weight", "offset": 6393872384, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 6441779200, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.33.ffn_norm.weight", "offset": 6489686016, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 6489702400, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 6491930624, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.33.attn_q.weight", "offset": 6509756416, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.33.attn_v.weight", "offset": 6527582208, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 6529810432, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6529826816, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.34.ffn_gate.weight", "offset": 6577733632, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 6625640448, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 6673547264, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 6673563648, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 6675791872, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.34.attn_q.weight", "offset": 6693617664, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.34.attn_v.weight", "offset": 6711443456, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 6713671680, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6713688064, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.35.ffn_gate.weight", "offset": 6761594880, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 6809501696, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.35.ffn_norm.weight", "offset": 6857408512, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 6857424896, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 6859653120, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.35.attn_q.weight", "offset": 6877478912, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.35.attn_v.weight", "offset": 6895304704, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.36.attn_norm.weight", "offset": 6897532928, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6897549312, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.36.ffn_gate.weight", "offset": 6945456128, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 6993362944, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 7041269760, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 7041286144, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 7043514368, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.36.attn_q.weight", "offset": 7061340160, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.36.attn_v.weight", "offset": 7079165952, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.37.attn_norm.weight", "offset": 7081394176, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7081410560, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.37.ffn_gate.weight", "offset": 7129317376, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 7177224192, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.37.ffn_norm.weight", "offset": 7225131008, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 7225147392, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 7227375616, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.37.attn_q.weight", "offset": 7245201408, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.37.attn_v.weight", "offset": 7263027200, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 7265255424, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7265271808, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.38.ffn_gate.weight", "offset": 7313178624, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 7361085440, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.38.ffn_norm.weight", "offset": 7408992256, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 7409008640, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 7411236864, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.38.attn_q.weight", "offset": 7429062656, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.38.attn_v.weight", "offset": 7446888448, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 7449116672, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7449133056, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.39.ffn_gate.weight", "offset": 7497039872, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 7544946688, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.39.ffn_norm.weight", "offset": 7592853504, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 7592869888, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 7595098112, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.39.attn_q.weight", "offset": 7612923904, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.39.attn_v.weight", "offset": 7630749696, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 7632977920, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 7632994304, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.40.ffn_gate.weight", "offset": 7680901120, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.40.ffn_up.weight", "offset": 7728807936, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.40.ffn_norm.weight", "offset": 7776714752, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 7776731136, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 7778959360, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.40.attn_q.weight", "offset": 7796785152, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.40.attn_v.weight", "offset": 7814610944, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.41.ffn_gate.weight", "offset": 7816839168, "shape": [ 4096, 11008 ], "size": 2228224, "type": 8 }, { "name": "blk.41.attn_k.weight", "offset": 7864745984, "shape": [ 4096, 512 ], "size": 47906816, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 7866974208, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.41.attn_q.weight", "offset": 7884800000, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.41.attn_v.weight", "offset": 7902625792, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "output.weight", "offset": 7904854016, "shape": [ 4096, 64000 ], "size": 2228224, "type": 8 }, { "name": "blk.41.attn_norm.weight", "offset": 8183382016, "shape": [ 4096 ], "size": 278528000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 8183398400, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.41.ffn_up.weight", "offset": 8231305216, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.41.ffn_norm.weight", "offset": 8279212032, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 8279228416, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 8279244800, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.42.ffn_gate.weight", "offset": 8327151616, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.42.ffn_up.weight", "offset": 8375058432, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.42.ffn_norm.weight", "offset": 8422965248, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 8422981632, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 8425209856, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.42.attn_q.weight", "offset": 8443035648, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.42.attn_v.weight", "offset": 8460861440, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.43.attn_norm.weight", "offset": 8463089664, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 8463106048, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.43.ffn_gate.weight", "offset": 8511012864, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.43.ffn_up.weight", "offset": 8558919680, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.43.ffn_norm.weight", "offset": 8606826496, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 8606842880, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 8609071104, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.43.attn_q.weight", "offset": 8626896896, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.43.attn_v.weight", "offset": 8644722688, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.44.attn_norm.weight", "offset": 8646950912, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 8646967296, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.44.ffn_gate.weight", "offset": 8694874112, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.44.ffn_up.weight", "offset": 8742780928, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.44.ffn_norm.weight", "offset": 8790687744, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 8790704128, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 8792932352, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.44.attn_q.weight", "offset": 8810758144, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.44.attn_v.weight", "offset": 8828583936, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.45.attn_norm.weight", "offset": 8830812160, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 8830828544, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.45.ffn_gate.weight", "offset": 8878735360, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.45.ffn_up.weight", "offset": 8926642176, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 8974548992, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 8974565376, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 8976793600, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.45.attn_q.weight", "offset": 8994619392, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.45.attn_v.weight", "offset": 9012445184, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.46.attn_norm.weight", "offset": 9014673408, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 9014689792, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.46.ffn_gate.weight", "offset": 9062596608, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.46.ffn_up.weight", "offset": 9110503424, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.46.ffn_norm.weight", "offset": 9158410240, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 9158426624, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 9160654848, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.46.attn_q.weight", "offset": 9178480640, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.46.attn_v.weight", "offset": 9196306432, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "blk.47.attn_norm.weight", "offset": 9198534656, "shape": [ 4096 ], "size": 2228224, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 9198551040, "shape": [ 11008, 4096 ], "size": 16384, "type": 8 }, { "name": "blk.47.ffn_gate.weight", "offset": 9246457856, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.47.ffn_up.weight", "offset": 9294364672, "shape": [ 4096, 11008 ], "size": 47906816, "type": 8 }, { "name": "blk.47.ffn_norm.weight", "offset": 9342271488, "shape": [ 4096 ], "size": 47906816, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 9342287872, "shape": [ 4096, 512 ], "size": 16384, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 9344516096, "shape": [ 4096, 4096 ], "size": 2228224, "type": 8 }, { "name": "blk.47.attn_q.weight", "offset": 9362341888, "shape": [ 4096, 4096 ], "size": 17825792, "type": 8 }, { "name": "blk.47.attn_v.weight", "offset": 9380167680, "shape": [ 4096, 512 ], "size": 17825792, "type": 8 }, { "name": "output_norm.weight", "offset": 9382395904, "shape": [ 4096 ], "size": 2228224, "type": 0 } ], "version": 3 }