phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 9, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 7 }, { "name": "blk.0.attn_norm.weight", "offset": 123126272, "shape": [ 5120 ], "size": 123125760, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 123146752, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 191959552, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 329585152, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 329605632, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.0.attn_qkv.weight", "offset": 349266432, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 378757632, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 378778112, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 447590912, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 585216512, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 585236992, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.1.attn_qkv.weight", "offset": 604897792, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 634388992, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 634409472, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 703222272, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 840847872, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 840868352, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.2.attn_qkv.weight", "offset": 860529152, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 890020352, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 890040832, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 958853632, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1096479232, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1096499712, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.3.attn_qkv.weight", "offset": 1116160512, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1145651712, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1145672192, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1214484992, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1352110592, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1352131072, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.4.attn_qkv.weight", "offset": 1371791872, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1401283072, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1401303552, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1470116352, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1607741952, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1607762432, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.5.attn_qkv.weight", "offset": 1627423232, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1656914432, "shape": [ 5120, 35840 ], "size": 29491200, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1794540032, "shape": [ 5120, 5120 ], "size": 137625600, "type": 7 }, { "name": "blk.6.attn_qkv.weight", "offset": 1814200832, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1843692032, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1843712512, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1912525312, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 2050150912, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2050171392, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.10.attn_qkv.weight", "offset": 2069832192, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 2099323392, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2099343872, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 2168156672, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 2305782272, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2305802752, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.11.attn_qkv.weight", "offset": 2325463552, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 2354954752, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2354975232, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 2423788032, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 2561413632, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2561434112, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.12.attn_qkv.weight", "offset": 2581094912, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 2610586112, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2610606592, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 2679419392, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 2817044992, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2817065472, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.13.attn_qkv.weight", "offset": 2836726272, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 2866217472, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2866237952, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 2935050752, "shape": [ 5120 ], "size": 68812800, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2935071232, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2935091712, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 3003904512, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 3141530112, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3141550592, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.7.attn_qkv.weight", "offset": 3161211392, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 3190702592, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3190723072, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 3259535872, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 3397161472, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3397181952, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.8.attn_qkv.weight", "offset": 3416842752, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 3446333952, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3446354432, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 3515167232, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 3652792832, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3652813312, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.9.attn_qkv.weight", "offset": 3672474112, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 3701965312, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3701985792, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 3770798592, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 3908424192, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3908444672, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.14.attn_qkv.weight", "offset": 3928105472, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 3957596672, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3957617152, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 4026429952, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 4164055552, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 4164076032, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.15.attn_qkv.weight", "offset": 4183736832, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 4213228032, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4213248512, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 4282061312, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 4419686912, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4419707392, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.16.attn_qkv.weight", "offset": 4439368192, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 4468859392, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4468879872, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 4537692672, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 4675318272, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4675338752, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.17.attn_qkv.weight", "offset": 4694999552, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 4724490752, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4724511232, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 4793324032, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 4930949632, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4930970112, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.18.attn_qkv.weight", "offset": 4950630912, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 4980122112, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4980142592, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 5048955392, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 5186580992, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 5186601472, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.19.attn_qkv.weight", "offset": 5206262272, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 5235753472, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5235773952, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 5304586752, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 5442212352, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5442232832, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.20.attn_qkv.weight", "offset": 5461893632, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 5491384832, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.21.attn_qkv.weight", "offset": 5511045632, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 5540536832, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5540557312, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 5609370112, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 5746995712, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5747016192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5747036672, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 5815849472, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 5953475072, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5953495552, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.22.attn_qkv.weight", "offset": 5973156352, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 6002647552, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6002668032, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 6071480832, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 6209106432, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 6209126912, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.23.attn_qkv.weight", "offset": 6228787712, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 6258278912, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6258299392, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 6327112192, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 6464737792, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 6464758272, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.24.attn_qkv.weight", "offset": 6484419072, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 6513910272, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6513930752, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 6582743552, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 6720369152, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6720389632, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.25.attn_qkv.weight", "offset": 6740050432, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 6769541632, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6769562112, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 6838374912, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 6976000512, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6976020992, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.26.attn_qkv.weight", "offset": 6995681792, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 7025172992, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7025193472, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 7094006272, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 7231631872, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 7231652352, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.27.attn_qkv.weight", "offset": 7251313152, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 7280804352, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.28.attn_qkv.weight", "offset": 7300465152, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 7329956352, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7329976832, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 7398789632, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 7536415232, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 7536435712, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7536456192, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 7605268992, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 7742894592, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7742915072, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.29.attn_qkv.weight", "offset": 7762575872, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 7792067072, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7792087552, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 7860900352, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 7998525952, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7998546432, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.30.attn_qkv.weight", "offset": 8018207232, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 8047698432, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8047718912, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 8116531712, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 8254157312, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 8254177792, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.31.attn_qkv.weight", "offset": 8273838592, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 8303329792, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 8303350272, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 8372163072, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 8509788672, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 8509809152, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.32.attn_qkv.weight", "offset": 8529469952, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.33.attn_norm.weight", "offset": 8558961152, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8558981632, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 8627794432, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 8765420032, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8765440512, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.33.attn_qkv.weight", "offset": 8785101312, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 8814592512, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8814612992, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 8883425792, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 9021051392, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 9021071872, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.34.attn_qkv.weight", "offset": 9040732672, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 9070223872, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.35.attn_qkv.weight", "offset": 9089884672, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "output.weight", "offset": 9119375872, "shape": [ 5120, 32064 ], "size": 29491200, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 9254044672, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 9254065152, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 9322877952, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 9460503552, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 9460524032, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 9460544512, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 9529357312, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 9666982912, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 9667003392, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.36.attn_qkv.weight", "offset": 9686664192, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 9716155392, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9716175872, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 9784988672, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 9922614272, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9922634752, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.37.attn_qkv.weight", "offset": 9942295552, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 9971786752, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9971807232, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 10040620032, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 10178245632, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 10178266112, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.38.attn_qkv.weight", "offset": 10197926912, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 10227418112, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 10227438592, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 10296251392, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 10433876992, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 10433897472, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.39.attn_qkv.weight", "offset": 10453558272, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "output_norm.weight", "offset": 10483049472, "shape": [ 5120 ], "size": 29491200, "type": 0 } ], "version": 3 }