yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 13, "general.name": "Yi-1.5-9B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 112640000, "shape": [ 4096 ], "size": 112640000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112656384, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 143654912, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 163028992, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 182403072, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 182419456, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 183320576, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 194854912, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 202063872, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 203505664, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 203522048, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 234520576, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 253894656, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 273268736, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 273285120, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 274186240, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 285720576, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 292929536, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 294371328, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 294387712, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 325386240, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 344760320, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 364134400, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 364150784, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 365051904, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 376586240, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 383795200, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 385236992, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 385253376, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 416251904, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 435625984, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 455000064, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 455016448, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 455917568, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 467451904, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 474660864, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 476102656, "shape": [ 4096, 11008 ], "size": 1441792, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 495476736, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 514850816, "shape": [ 4096, 512 ], "size": 19374080, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 515751936, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 527286272, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 534495232, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 535937024, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 535953408, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 566951936, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 586326016, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 605700096, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 605716480, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 606617600, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 618151936, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 625360896, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 626802688, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 626819072, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 657817600, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 677191680, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 696565760, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 696582144, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 697483264, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 709017600, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.3.attn_v.weight", "offset": 716226560, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 717668352, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 717684736, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 748683264, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 768057344, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 787431424, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 787447808, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 788348928, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 799883264, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 807092224, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 808534016, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 808550400, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 839548928, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 858923008, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 878297088, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 878313472, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 879214592, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 890748928, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 897957888, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 899399680, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 899416064, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 930414592, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 949788672, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 969162752, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 969179136, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 970080256, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 981614592, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 988823552, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 990265344, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 990281728, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 1021280256, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1040654336, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1060028416, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1060044800, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1060945920, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 1072480256, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 1079689216, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1081131008, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1081147392, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1112145920, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1131520000, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1150894080, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1150910464, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1151811584, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 1163345920, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 1170554880, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1171996672, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1172013056, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1203011584, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1222385664, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1241759744, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1241776128, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1242677248, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1254211584, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.9.attn_v.weight", "offset": 1261420544, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1262862336, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1262878720, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 1293877248, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 1293893632, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1293910016, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1324908544, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1344282624, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1363656704, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1363673088, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1364574208, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 1376108544, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 1383317504, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 1384759296, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1384775680, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1415774208, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1435148288, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 1454522368, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1454538752, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 1455439872, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 1466974208, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1474183168, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 1475624960, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1475641344, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 1506639872, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1526013952, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 1545388032, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1545404416, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1546305536, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 1557839872, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1565048832, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 1566490624, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1566507008, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 1597505536, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 1616879616, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 1636253696, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1636270080, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 1637171200, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 1648705536, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1655914496, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 1657356288, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1657372672, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 1688371200, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 1707745280, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 1727119360, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1727135744, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 1728036864, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 1739571200, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 1746780160, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 1748221952, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1748238336, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 1779236864, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1798610944, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 1817985024, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1818001408, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 1818902528, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 1830436864, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1837645824, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 1839087616, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1839104000, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 1870102528, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 1889476608, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 1908850688, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1908867072, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 1909768192, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 1921302528, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 1928511488, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 1929953280, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1929969664, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 1960968192, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 1980342272, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 1999716352, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 1999732736, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 2000633856, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2012168192, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 2019377152, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2020818944, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2020835328, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 2051833856, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2071207936, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 2090582016, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2090598400, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2091499520, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 2103033856, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 2110242816, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 2111684608, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2111700992, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2142699520, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2162073600, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 2181447680, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2181464064, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2182365184, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 2193899520, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2201108480, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 2202550272, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2202566656, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 2233565184, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2252939264, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 2272313344, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2272329728, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 2273230848, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 2284765184, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2291974144, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 2293415936, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2293432320, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 2324430848, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2343804928, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 2363179008, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2363195392, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 2364096512, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.40.attn_nor", "offset": 2375630848, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2382839808, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 2384281600, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2384297984, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 2415296512, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2434670592, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 2454044672, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2454061056, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 2454962176, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 2466496512, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2473705472, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 2475147264, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2475163648, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 2506162176, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2525536256, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 2544910336, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2544926720, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 2545827840, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 2557362176, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2564571136, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.27.attn_k.weight", "offset": 2566012928, "shape": [ 4096, 512 ], "size": 1441792, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 2566914048, "shape": [ 4096, 4096 ], "size": 901120, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2574123008, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 2575564800, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2575581184, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 2606579712, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2625953792, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 2645327872, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 2645344256, "shape": [ 4096, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 2656878592, "shape": [ 4096 ], "size": 11534336, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2656894976, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 2687893504, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2707267584, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 2726641664, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 2726658048, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 2727559168, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 2739093504, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 2746302464, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 2747744256, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 2747760640, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 2778759168, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 2798133248, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 2817507328, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 2817523712, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 2818424832, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 2829959168, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 2837168128, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 2838609920, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2838626304, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 2869624832, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 2888998912, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 2908372992, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 2908389376, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 2909290496, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 2920824832, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 2928033792, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 2929475584, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 2929491968, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 2960490496, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 2979864576, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 2999238656, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 2999255040, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 3000156160, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 3011690496, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 3018899456, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 3020341248, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3020357632, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 3051356160, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 3070730240, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 3090104320, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 3090120704, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 3091021824, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 3102556160, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 3109765120, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 3111206912, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3111223296, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 3142221824, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 3161595904, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 3180969984, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3180986368, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 3181887488, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 3193421824, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 3200630784, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 3202072576, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3202088960, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 3233087488, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 3252461568, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 3271835648, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3271852032, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 3272753152, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 3284287488, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 3291496448, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 3292938240, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3292954624, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 3323953152, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 3343327232, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 3362701312, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3362717696, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 3363618816, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 3375153152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 3382362112, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 3383803904, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3383820288, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 3414818816, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 3434192896, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 3453566976, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3453583360, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 3454484480, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 3466018816, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 3473227776, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 3474669568, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3474685952, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 3505684480, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 3525058560, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 3544432640, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 3544449024, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 3545350144, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 3556884480, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 3564093440, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 3565535232, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 3565551616, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 3596550144, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 3615924224, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 3635298304, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 3635314688, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 3636215808, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 3647750144, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 3654959104, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 3656400896, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 3656417280, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 3687415808, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 3706789888, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 3726163968, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 3726180352, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 3727081472, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 3738615808, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 3745824768, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 3747266560, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 3747282944, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 3778281472, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 3797655552, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 3817029632, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 3817046016, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 3817947136, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 3829481472, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 3836690432, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 3838132224, "shape": [ 4096, 11008 ], "size": 1441792, "type": 11 }, { "name": "blk.41.attn_k.weight", "offset": 3857506304, "shape": [ 4096, 512 ], "size": 19374080, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 3858407424, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 3869941760, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 3877150720, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "output.weight", "offset": 3878592512, "shape": [ 4096, 64000 ], "size": 1441792, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 4093632512, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4093648896, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 4124647424, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 4144021504, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 4144037888, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 4144054272, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.42.ffn_gate.weight", "offset": 4175052800, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.42.ffn_up.weight", "offset": 4194426880, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 4213800960, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 4213817344, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 4214718464, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.42.attn_q.weight", "offset": 4226252800, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.42.attn_v.weight", "offset": 4233461760, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 4234903552, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 4234919936, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.43.ffn_gate.weight", "offset": 4265918464, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.43.ffn_up.weight", "offset": 4285292544, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 4304666624, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 4304683008, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 4305584128, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.43.attn_q.weight", "offset": 4317118464, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.43.attn_v.weight", "offset": 4324327424, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.44.attn_norm.weight", "offset": 4325769216, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 4325785600, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.44.ffn_gate.weight", "offset": 4356784128, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.44.ffn_up.weight", "offset": 4376158208, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 4395532288, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 4395548672, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 4396449792, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.44.attn_q.weight", "offset": 4407984128, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.44.attn_v.weight", "offset": 4415193088, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 4416634880, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 4416651264, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.45.ffn_gate.weight", "offset": 4447649792, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.45.ffn_up.weight", "offset": 4467023872, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 4486397952, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 4486414336, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 4487315456, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.45.attn_q.weight", "offset": 4498849792, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.45.attn_v.weight", "offset": 4506058752, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 4507500544, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 4507516928, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.46.ffn_gate.weight", "offset": 4538515456, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.46.ffn_up.weight", "offset": 4557889536, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 4577263616, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 4577280000, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 4578181120, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.46.attn_q.weight", "offset": 4589715456, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.46.attn_v.weight", "offset": 4596924416, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 4598366208, "shape": [ 4096 ], "size": 1441792, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 4598382592, "shape": [ 11008, 4096 ], "size": 16384, "type": 13 }, { "name": "blk.47.ffn_gate.weight", "offset": 4629381120, "shape": [ 4096, 11008 ], "size": 30998528, "type": 11 }, { "name": "blk.47.ffn_up.weight", "offset": 4648755200, "shape": [ 4096, 11008 ], "size": 19374080, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 4668129280, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 4668145664, "shape": [ 4096, 512 ], "size": 16384, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 4669046784, "shape": [ 4096, 4096 ], "size": 901120, "type": 13 }, { "name": "blk.47.attn_q.weight", "offset": 4680581120, "shape": [ 4096, 4096 ], "size": 11534336, "type": 11 }, { "name": "blk.47.attn_v.weight", "offset": 4687790080, "shape": [ 4096, 512 ], "size": 7208960, "type": 13 }, { "name": "output_norm.weight", "offset": 4689231872, "shape": [ 4096 ], "size": 1441792, "type": 0 } ], "version": 3 }