yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 12, "general.name": "Yi-1.5-9B", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 112640000, "shape": [ 4096 ], "size": 112640000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112656384, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 143654912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 163028992, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 182403072, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 182419456, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 183320576, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 192757760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 199966720, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 201408512, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 201424896, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 232423424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 251797504, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 271171584, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 271187968, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 272089088, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 281526272, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 288735232, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 290177024, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 290193408, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 321191936, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 340566016, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 359940096, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 359956480, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 360857600, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 370294784, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 377503744, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 378683392, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 378699776, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 404062208, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 423436288, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 442810368, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 442826752, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 443727872, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 453165056, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 460374016, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 461553664, "shape": [ 4096, 11008 ], "size": 1179648, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 480927744, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 500301824, "shape": [ 4096, 512 ], "size": 19374080, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 501202944, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 510640128, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 517849088, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 519028736, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 519045120, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 544407552, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 563781632, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 583155712, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 583172096, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 584073216, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 593510400, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 600719360, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 601899008, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 601915392, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 627277824, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 646651904, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 666025984, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 666042368, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 666943488, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 676380672, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 683589632, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 684769280, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 684785664, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 710148096, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 729522176, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 748896256, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 748912640, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 749813760, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 759250944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 766459904, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 767639552, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 767655936, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 793018368, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 812392448, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 831766528, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 831782912, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 832684032, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 842121216, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 849330176, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 850509824, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 850526208, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 875888640, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 895262720, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 914636800, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 914653184, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 915554304, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 924991488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 932200448, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 933380096, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 933396480, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 958758912, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 978132992, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 997507072, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 997523456, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 998424576, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 1007861760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 1015070720, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1016250368, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1016266752, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1041629184, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1061003264, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1080377344, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1080393728, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1081294848, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 1090732032, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 1097940992, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1099120640, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1099137024, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 1124499456, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1143873536, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1163247616, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1163264000, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1164165120, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 1173602304, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 1180811264, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1181990912, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1182007296, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1207369728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1207386112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1207402496, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1232764928, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1252139008, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1271513088, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1271529472, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1272430592, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 1281867776, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 1289076736, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1290256384, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1290272768, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1315635200, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1335009280, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 1354383360, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1354399744, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 1355300864, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1364738048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1371947008, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1373126656, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1373143040, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1398505472, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1417879552, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 1437253632, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1437270016, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1438171136, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1447608320, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1454817280, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 1455996928, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1456013312, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1481375744, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 1500749824, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 1520123904, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1520140288, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 1521041408, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 1530478592, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1537687552, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1538867200, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1538883584, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 1564246016, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 1583620096, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 1602994176, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1603010560, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 1603911680, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 1613348864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 1620557824, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 1621737472, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1621753856, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 1647116288, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1666490368, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 1685864448, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1685880832, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 1686781952, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 1696219136, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1703428096, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 1704607744, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1704624128, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 1729986560, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 1749360640, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 1768734720, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1768751104, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 1769652224, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 1779089408, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 1786298368, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 1787478016, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1787494400, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 1812856832, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 1832230912, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 1851604992, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 1851621376, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 1852522496, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 1861959680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 1869168640, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 1870348288, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1870364672, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 1895727104, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 1915101184, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 1934475264, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 1934491648, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 1935392768, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 1944829952, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 1952038912, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 1953218560, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1953234944, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 1978597376, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 1997971456, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 2017345536, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2017361920, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2018263040, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 2027700224, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2034909184, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2036088832, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2036105216, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2061467648, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2080841728, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 2100215808, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2100232192, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 2101133312, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 2110570496, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2117779456, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2118959104, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2118975488, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2144337920, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2163712000, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 2183086080, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2183102464, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 2184003584, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 2193440768, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2200649728, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2201829376, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2201845760, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2227208192, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2246582272, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 2265956352, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2265972736, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 2266873856, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 2276311040, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2283520000, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2284699648, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2284716032, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 2310078464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2329452544, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 2348826624, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2348843008, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 2349744128, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 2359181312, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2366390272, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.27.attn_k.weight", "offset": 2367569920, "shape": [ 4096, 512 ], "size": 1179648, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 2368471040, "shape": [ 4096, 4096 ], "size": 901120, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2375680000, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 2376859648, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2376876032, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 2402238464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2421612544, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 2440986624, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 2441003008, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 2450440192, "shape": [ 4096 ], "size": 9437184, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2450456576, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 2475819008, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2495193088, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 2514567168, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 2514583552, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 2515484672, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 2524921856, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 2532130816, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 2533310464, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 2533326848, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 2558689280, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 2578063360, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 2597437440, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 2597453824, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 2598354944, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 2607792128, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 2615001088, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 2616180736, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2616197120, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 2641559552, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 2660933632, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 2680307712, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 2680324096, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 2681225216, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 2690662400, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 2697871360, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 2699051008, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 2699067392, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 2724429824, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 2743803904, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 2763177984, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 2763194368, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 2764095488, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 2773532672, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 2780741632, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 2781921280, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 2781937664, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 2807300096, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 2826674176, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 2846048256, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 2846064640, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 2846965760, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 2856402944, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 2863611904, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 2864791552, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 2864807936, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 2890170368, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 2909544448, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 2928918528, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 2928934912, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 2929836032, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 2939273216, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 2946482176, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 2947661824, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 2947678208, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 2973040640, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 2992414720, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 3011788800, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3011805184, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 3012706304, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 3022143488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 3029352448, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 3030532096, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3030548480, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 3055910912, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 3075284992, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 3094659072, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3094675456, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 3095576576, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 3105013760, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 3112222720, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 3113402368, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3113418752, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 3138781184, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 3158155264, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 3177529344, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3177545728, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 3178446848, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 3187884032, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 3195092992, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 3196272640, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3196289024, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 3221651456, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 3241025536, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 3260399616, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 3260416000, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 3261317120, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 3270754304, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 3277963264, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 3279142912, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 3279159296, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 3304521728, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 3323895808, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 3343269888, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 3343286272, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 3344187392, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 3353624576, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 3360833536, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 3362013184, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 3362029568, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 3387392000, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 3406766080, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 3426140160, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 3426156544, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 3427057664, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 3436494848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 3443703808, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 3444883456, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 3444899840, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 3470262272, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 3489636352, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 3509010432, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 3509026816, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 3509927936, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 3519365120, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 3526574080, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 3527753728, "shape": [ 4096, 11008 ], "size": 1179648, "type": 11 }, { "name": "blk.41.attn_k.weight", "offset": 3547127808, "shape": [ 4096, 512 ], "size": 19374080, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 3548028928, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 3557466112, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 3564675072, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "output.weight", "offset": 3565854720, "shape": [ 4096, 64000 ], "size": 1179648, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 3780894720, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 3780911104, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 3806273536, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 3825647616, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 3825664000, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 3825680384, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.42.ffn_gate.weight", "offset": 3851042816, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 3870416896, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 3889790976, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 3889807360, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 3890708480, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.42.attn_q.weight", "offset": 3900145664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 3907354624, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 3908534272, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 3908550656, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.43.ffn_gate.weight", "offset": 3933913088, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 3953287168, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 3972661248, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 3972677632, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 3973578752, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.43.attn_q.weight", "offset": 3983015936, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 3990224896, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.44.attn_norm.weight", "offset": 3991404544, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 3991420928, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.44.ffn_gate.weight", "offset": 4016783360, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 4036157440, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 4055531520, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 4055547904, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 4056449024, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.44.attn_q.weight", "offset": 4065886208, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 4073095168, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 4074274816, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 4074291200, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.45.ffn_gate.weight", "offset": 4099653632, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 4119027712, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 4138401792, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 4138418176, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 4139319296, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.45.attn_q.weight", "offset": 4148756480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 4155965440, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 4157145088, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 4157161472, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.46.ffn_gate.weight", "offset": 4182523904, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 4201897984, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 4221272064, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 4221288448, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 4222189568, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.46.attn_q.weight", "offset": 4231626752, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 4238835712, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 4240015360, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 4240031744, "shape": [ 11008, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.47.ffn_gate.weight", "offset": 4265394176, "shape": [ 4096, 11008 ], "size": 25362432, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 4284768256, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 4304142336, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 4304158720, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 4305059840, "shape": [ 4096, 4096 ], "size": 901120, "type": 12 }, { "name": "blk.47.attn_q.weight", "offset": 4314497024, "shape": [ 4096, 4096 ], "size": 9437184, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 4321705984, "shape": [ 4096, 512 ], "size": 7208960, "type": 12 }, { "name": "output_norm.weight", "offset": 4322885632, "shape": [ 4096 ], "size": 1179648, "type": 0 } ], "version": 3 }