The 7B model released by Mistral AI, updated to version 0.3.

7B

2.3M Pulls Updated 3 weeks ago

84 Tags

ff82381e2bea · 4.1GB
{ "metadata": { "general.architecture": "llama", "general.file_type": 2, "general.name": "Mistral-7B-Instruct-v0.3", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 8, "llama.attention.layer_norm_rms_epsilon": 0.00001, "llama.block_count": 32, "llama.context_length": 32768, "llama.embedding_length": 4096, "llama.feed_forward_length": 14336, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 1000000, "llama.vocab_size": 32768, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32768 values)", "tokenizer.ggml.token_type": "... (32768 values)", "tokenizer.ggml.tokens": "... (32768 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 7248023552, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 75497472, "shape": [ 4096 ], "size": 75497472, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 75513856, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 108544000, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 141574144, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 174604288, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 174620672, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 176979968, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 186417152, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 195854336, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 198213632, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 198230016, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 231260160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 264290304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 297320448, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 297336832, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 299696128, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 309133312, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 318570496, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 320929792, "shape": [ 4096, 14336 ], "size": 2359296, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 353959936, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.10.attn_k.weight", "offset": 386990080, "shape": [ 4096, 1024 ], "size": 33030144, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 389349376, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 398786560, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 408223744, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 410583040, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 410599424, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 443629568, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 476659712, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 509689856, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 509706240, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 512065536, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 521502720, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 530939904, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 533299200, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 533315584, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 566345728, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 599375872, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 632406016, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 632422400, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 634781696, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 644218880, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 653656064, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 656015360, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 656031744, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 689061888, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 722092032, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 755122176, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 755138560, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 757497856, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 766935040, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 776372224, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 778731520, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 778747904, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 811778048, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 844808192, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 877838336, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 877854720, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 880214016, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 889651200, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 899088384, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 901447680, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 901464064, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 934494208, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 967524352, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1000554496, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1000570880, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1002930176, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 1012367360, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 1021804544, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 1024163840, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1024180224, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 1057210368, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1090240512, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1123270656, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1123287040, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 1125646336, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 1135083520, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 1144520704, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1146880000, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1146896384, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 1179926528, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 1212956672, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 1245986816, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1246003200, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 1248362496, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 1257799680, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 1267236864, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 1269596160, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1269612544, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 1302642688, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 1335672832, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 1368702976, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1368719360, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 1371078656, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 1380515840, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 1389953024, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1392312320, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1392328704, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1425358848, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.11.attn_norm.weight", "offset": 1425375232, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1425391616, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1458421760, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1491451904, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1524482048, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1524498432, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 1526857728, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 1536294912, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 1545732096, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1548091392, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1548107776, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 1581137920, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 1614168064, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1647198208, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1647214592, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 1649573888, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 1659011072, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 1668448256, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1670807552, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1670823936, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 1703854080, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 1736884224, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 1769914368, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1769930752, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 1772290048, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 1781727232, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 1791164416, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 1793523712, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1793540096, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 1826570240, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 1859600384, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 1892630528, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1892646912, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 1895006208, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 1904443392, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 1913880576, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 1916239872, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1916256256, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 1949286400, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 1982316544, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 2015346688, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2015363072, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 2017722368, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 2027159552, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 2036596736, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 2038956032, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2038972416, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 2072002560, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 2105032704, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 2138062848, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2138079232, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 2140438528, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 2149875712, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 2159312896, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 2161672192, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2161688576, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 2194718720, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 2227748864, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 2260779008, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2260795392, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 2263154688, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 2272591872, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 2282029056, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 2284388352, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2284404736, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 2317434880, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 2350465024, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 2383495168, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2383511552, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 2385870848, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 2395308032, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 2404745216, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 2407104512, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2407120896, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 2440151040, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 2473181184, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 2506211328, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2506227712, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 2508587008, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 2518024192, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 2527461376, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 2529820672, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2529837056, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 2562867200, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 2595897344, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 2628927488, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2628943872, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 2631303168, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 2640740352, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 2650177536, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 2652536832, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2652553216, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 2685583360, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 2718613504, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 2751643648, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2751660032, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 2754019328, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 2763456512, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 2772893696, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_k.weight", "offset": 2775252992, "shape": [ 4096, 1024 ], "size": 2359296, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 2777612288, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 2787049472, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 2796486656, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "output.weight", "offset": 2798845952, "shape": [ 4096, 32768 ], "size": 2359296, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 2908946432, "shape": [ 4096 ], "size": 110100480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2908962816, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 2941992960, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 2975023104, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 3008053248, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3008069632, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3008086016, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 3041116160, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 3074146304, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 3107176448, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3107192832, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 3109552128, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 3118989312, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 3128426496, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 3130785792, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3130802176, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 3163832320, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 3196862464, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 3229892608, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "\ufffd\ufffd\ufffdH\ufffd؈\ufffd\ufffd\ufffd\u0012H\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd", "offset": 3229908992, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 3232268288, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 3241705472, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 3251142656, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 3253501952, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3253518336, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 3286548480, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 3319578624, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 3352608768, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3352625152, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 3354984448, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 3364421632, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 3373858816, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 3376218112, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3376234496, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 3409264640, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 3442294784, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 3475324928, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3475341312, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 3477700608, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 3487137792, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 3496574976, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 3498934272, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3498950656, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 3531980800, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 3565010944, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 3598041088, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3598057472, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 3600416768, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 3609853952, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 3619291136, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 3621650432, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3621666816, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 3654696960, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 3687727104, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 3720757248, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3720773632, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 3723132928, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 3732570112, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 3742007296, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 3744366592, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3744382976, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 3777413120, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 3810443264, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 3843473408, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3843489792, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 3845849088, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 3855286272, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 3864723456, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 3867082752, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3867099136, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 3900129280, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 3933159424, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 3966189568, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3966205952, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 3968565248, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 3978002432, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 3987439616, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 3989798912, "shape": [ 4096 ], "size": 2359296, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3989815296, "shape": [ 14336, 4096 ], "size": 16384, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 4022845440, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 4055875584, "shape": [ 4096, 14336 ], "size": 33030144, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 4088905728, "shape": [ 4096 ], "size": 33030144, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 4088922112, "shape": [ 4096, 1024 ], "size": 16384, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 4091281408, "shape": [ 4096, 4096 ], "size": 2359296, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 4100718592, "shape": [ 4096, 4096 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 4110155776, "shape": [ 4096, 1024 ], "size": 9437184, "type": 2 }, { "name": "output_norm.weight", "offset": 4112515072, "shape": [ 4096 ], "size": 2359296, "type": 0 } ], "version": 3 }