wizard-math

{ "metadata": { "general.architecture": "llama", "general.file_type": 14, "general.name": "WizardLM", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 32, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 2048, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.scores": "... (32001 values)", "tokenizer.ggml.token_type": "... (32001 values)", "tokenizer.ggml.tokens": "... (32001 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 6738423808, "tensors": [ { "name": "blk.0.attn_q.weight", "offset": 73730304, "shape": [ 4096, 4096 ], "size": 73730304, "type": 12 }, { "name": "blk.0.attn_k.weight", "offset": 83167488, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 92604672, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 104139008, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.0.ffn_gate.weight", "offset": 113576192, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 138938624, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.0.ffn_down.weight", "offset": 164301056, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 195299584, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 195315968, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 195332352, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.1.attn_k.weight", "offset": 204769536, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 214206720, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 225741056, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 235178240, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 260540672, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.1.ffn_down.weight", "offset": 285903104, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 316901632, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 316918016, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 316934400, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": ".ffn_gate.weight\u0002\u0000\u0000", "offset": 326371584, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.2.attn_v.weight", "offset": 335808768, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 347343104, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 356780288, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 382142720, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.2.ffn_down.weight", "offset": 407505152, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 438503680, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 438520064, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 438536448, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.3.attn_k.weight", "offset": 447973632, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.3.attn_v.weight", "offset": 457410816, "shape": [ 4096, 4096 ], "size": 9437184, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 468945152, "shape": [ 4096, 4096 ], "size": 11534336, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 478382336, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 503744768, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.3.ffn_down.weight", "offset": 529107200, "shape": [ 11008, 4096 ], "size": 25362432, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 560105728, "shape": [ 4096 ], "size": 30998528, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 560122112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 560138496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.4.attn_k.weight", "offset": 569575680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 579012864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 588450048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 597887232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 623249664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.4.ffn_down.weight", "offset": 648612096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 673974528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 673990912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 674007296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.5.attn_k.weight", "offset": 683444480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 692881664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 702318848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 711756032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 737118464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.5.ffn_down.weight", "offset": 762480896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 787843328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 787859712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 787876096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.6.attn_k.weight", "offset": 797313280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 806750464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 816187648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 825624832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 850987264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.6.ffn_down.weight", "offset": 876349696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 901712128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 901728512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 901744896, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.7.attn_k.weight", "offset": 911182080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 920619264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 930056448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 939493632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 964856064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.7.ffn_down.weight", "offset": 990218496, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 1015580928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1015597312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1015613696, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.8.attn_k.weight", "offset": 1025050880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 1034488064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 1043925248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1053362432, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 1078724864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.8.ffn_down.weight", "offset": 1104087296, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1129449728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1129466112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1129482496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.9.attn_k.weight", "offset": 1138919680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_v.weight", "offset": 1148356864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 1157794048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 1167231232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1192593664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.9.ffn_down.weight", "offset": 1217956096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1243318528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1243334912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1243351296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 1252788480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1262225664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1271662848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1281100032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1306462464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.10.ffn_down.weight", "offset": 1331824896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1357187328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.10.ffn_norm.weight", "offset": 1357203712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1357220096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.11.attn_k.weight", "offset": 1366657280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1376094464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1385531648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1394968832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1420331264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.11.ffn_down.weight", "offset": 1445693696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1471056128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1471072512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1471088896, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1480526080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1489963264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1499400448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1508837632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1534200064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.12.ffn_down.weight", "offset": 1559562496, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1584924928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.12.ffn_norm.weight", "offset": 1584941312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1584957696, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.13.attn_k.weight", "offset": 1594394880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1603832064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1613269248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1622706432, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1648068864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.13.ffn_down.weight", "offset": 1673431296, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1698793728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.13.ffn_norm.weight", "offset": 1698810112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1698826496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.14.attn_k.weight", "offset": 1708263680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1717700864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1727138048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1736575232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1761937664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.14.ffn_down.weight", "offset": 1787300096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1812662528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.14.ffn_norm.weight", "offset": 1812678912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1812695296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.15.attn_k.weight", "offset": 1822132480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 1831569664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1841006848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1850444032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 1875806464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.15.ffn_down.weight", "offset": 1901168896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1926531328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1926547712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1926564096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.16.attn_k.weight", "offset": 1936001280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 1945438464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1954875648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1964312832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 1989675264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.16.ffn_down.weight", "offset": 2015037696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2040400128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.16.ffn_norm.weight", "offset": 2040416512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2040432896, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.17.attn_k.weight", "offset": 2049870080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 2059307264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2068744448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "Iv\ufffd\ufffdtH\ufffdC\ufffd`ej'\ufffd\ufffd\ufffd\ufffd\t\ufffd\ufffd\ufffd\u0017", "offset": 2078181632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2103544064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.17.ffn_down.weight", "offset": 2128906496, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2154268928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2154285312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2154301696, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 2163738880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 2173176064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2182613248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2192050432, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2217412864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.18.ffn_down.weight", "offset": 2242775296, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 2268137728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.18.ffn_norm.weight", "offset": 2268154112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2268170496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.19.attn_k.weight", "offset": 2277607680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 2287044864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2296482048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2305919232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2331281664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.19.ffn_down.weight", "offset": 2356644096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2382006528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2382022912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2382039296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.20.attn_k.weight", "offset": 2391476480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 2400913664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2410350848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2419788032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 2445150464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.20.ffn_down.weight", "offset": 2470512896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2495875328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2495891712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2495908096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.21.attn_k.weight", "offset": 2505345280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 2514782464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2524219648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 2533656832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 2559019264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.21.ffn_down.weight", "offset": 2584381696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2609744128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2609760512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2609776896, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.22.attn_k.weight", "offset": 2619214080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 2628651264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2638088448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2647525632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 2672888064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.22.ffn_down.weight", "offset": 2698250496, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2723612928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2723629312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2723645696, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.23.attn_k.weight", "offset": 2733082880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 2742520064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2751957248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2761394432, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 2786756864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.23.ffn_down.weight", "offset": 2812119296, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2837481728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2837498112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2837514496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.24.attn_k.weight", "offset": 2846951680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 2856388864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2865826048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2875263232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 2900625664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.24.ffn_down.weight", "offset": 2925988096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2951350528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2951366912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2951383296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.25.attn_k.weight", "offset": 2960820480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 2970257664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 2979694848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2989132032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 3014494464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.25.ffn_down.weight", "offset": 3039856896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 3065219328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3065235712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 3065252096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.26.attn_k.weight", "offset": 3074689280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 3084126464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 3093563648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 3103000832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 3128363264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.26.ffn_down.weight", "offset": 3153725696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 3179088128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3179104512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 3179120896, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.27.attn_k.weight", "offset": 3188558080, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 3197995264, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 3207432448, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 3216869632, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 3242232064, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.27.ffn_down.weight", "offset": 3267594496, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3292956928, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3292973312, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.28.attn_q.weight", "offset": 3292989696, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.28.attn_k.weight", "offset": 3302426880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 3311864064, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 3321301248, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 3330738432, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 3356100864, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.28.ffn_down.weight", "offset": 3381463296, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 3406825728, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3406842112, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.29.attn_q.weight", "offset": 3406858496, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.29.attn_k.weight", "offset": 3416295680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 3425732864, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 3435170048, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 3444607232, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 3469969664, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.29.ffn_down.weight", "offset": 3495332096, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 3520694528, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3520710912, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.30.attn_q.weight", "offset": 3520727296, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.30.attn_k.weight", "offset": 3530164480, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 3539601664, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3549038848, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 3558476032, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 3583838464, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.30.ffn_down.weight", "offset": 3609200896, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 3634563328, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3634579712, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.31.attn_q.weight", "offset": 3634596096, "shape": [ 4096, 4096 ], "size": 16384, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 3644033280, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 3653470464, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 3662907648, "shape": [ 4096, 4096 ], "size": 9437184, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 3672344832, "shape": [ 4096, 11008 ], "size": 9437184, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 3697707264, "shape": [ 4096, 11008 ], "size": 25362432, "type": 12 }, { "name": "blk.31.ffn_down.weight", "offset": 3723069696, "shape": [ 11008, 4096 ], "size": 25362432, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 3748432128, "shape": [ 4096 ], "size": 25362432, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3748448512, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output_norm.weight", "offset": 3748464896, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "output.weight", "offset": 3748481280, "shape": [ 4096, 32001 ], "size": 16384, "type": 14 } ], "version": 2 }