phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 1, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 328335360, "shape": [ 5120 ], "size": 328335360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 328355840, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 511856640, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 878858240, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 878878720, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.0.attn_qkv.weight", "offset": 931307520, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 1009950720, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1009971200, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 1193472000, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 1560473600, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 1560494080, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.1.attn_qkv.weight", "offset": 1612922880, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1691566080, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1691586560, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 1875087360, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 2242088960, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 2242109440, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.2.attn_qkv.weight", "offset": 2294538240, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 2373181440, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2373201920, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 2556702720, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 2923704320, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 2923724800, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.3.attn_qkv.weight", "offset": 2976153600, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3054796800, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3054817280, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3238318080, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 3605319680, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 3605340160, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.4.attn_qkv.weight", "offset": 3657768960, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 3736412160, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3736432640, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 3919933440, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 4286935040, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 4286955520, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.5.attn_qkv.weight", "offset": 4339384320, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 4418027520, "shape": [ 5120, 35840 ], "size": 78643200, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 4785029120, "shape": [ 5120, 5120 ], "size": 367001600, "type": 1 }, { "name": "blk.6.attn_qkv.weight", "offset": 4837457920, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 4916101120, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4916121600, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 5099622400, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 5466624000, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 5466644480, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.10.attn_qkv.weight", "offset": 5519073280, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 5597716480, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5597736960, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 5781237760, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 6148239360, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 6148259840, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.11.attn_qkv.weight", "offset": 6200688640, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 6279331840, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 6279352320, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 6462853120, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 6829854720, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 6829875200, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.12.attn_qkv.weight", "offset": 6882304000, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 6960947200, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6960967680, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 7144468480, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 7511470080, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 7511490560, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.13.attn_qkv.weight", "offset": 7563919360, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 7642562560, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 7642583040, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 7826083840, "shape": [ 5120 ], "size": 183500800, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 7826104320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 7826124800, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 8009625600, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 8376627200, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 8376647680, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.7.attn_qkv.weight", "offset": 8429076480, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 8507719680, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 8507740160, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 8691240960, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 9058242560, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 9058263040, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.8.attn_qkv.weight", "offset": 9110691840, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 9189335040, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 9189355520, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 9372856320, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 9739857920, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 9739878400, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.9.attn_qkv.weight", "offset": 9792307200, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 9870950400, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 9870970880, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 10054471680, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 10421473280, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 10421493760, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.14.attn_qkv.weight", "offset": 10473922560, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 10552565760, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 10552586240, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 10736087040, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 11103088640, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 11103109120, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.15.attn_qkv.weight", "offset": 11155537920, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 11234181120, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 11234201600, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 11417702400, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 11784704000, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 11784724480, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.16.attn_qkv.weight", "offset": 11837153280, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 11915796480, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 11915816960, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 12099317760, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 12466319360, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 12466339840, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.17.attn_qkv.weight", "offset": 12518768640, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 12597411840, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 12597432320, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 12780933120, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 13147934720, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 13147955200, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.18.attn_qkv.weight", "offset": 13200384000, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 13279027200, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 13279047680, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 13462548480, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 13829550080, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 13829570560, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.19.attn_qkv.weight", "offset": 13881999360, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 13960642560, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 13960663040, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 14144163840, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 14511165440, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 14511185920, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.20.attn_qkv.weight", "offset": 14563614720, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 14642257920, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.21.attn_qkv.weight", "offset": 14694686720, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 14773329920, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 14773350400, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 14956851200, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 15323852800, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 15323873280, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 15323893760, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 15507394560, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 15874396160, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 15874416640, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.22.attn_qkv.weight", "offset": 15926845440, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 16005488640, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 16005509120, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 16189009920, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 16556011520, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 16556032000, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.23.attn_qkv.weight", "offset": 16608460800, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 16687104000, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 16687124480, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 16870625280, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 17237626880, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 17237647360, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.24.attn_qkv.weight", "offset": 17290076160, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 17368719360, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 17368739840, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 17552240640, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 17919242240, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 17919262720, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.25.attn_qkv.weight", "offset": 17971691520, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 18050334720, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 18050355200, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 18233856000, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 18600857600, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 18600878080, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.26.attn_qkv.weight", "offset": 18653306880, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 18731950080, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 18731970560, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 18915471360, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 19282472960, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 19282493440, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.27.attn_qkv.weight", "offset": 19334922240, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 19413565440, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.28.attn_qkv.weight", "offset": 19465994240, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 19544637440, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 19544657920, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 19728158720, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 20095160320, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 20095180800, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 20095201280, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 20278702080, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 20645703680, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 20645724160, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.29.attn_qkv.weight", "offset": 20698152960, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 20776796160, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 20776816640, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 20960317440, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 21327319040, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 21327339520, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.30.attn_qkv.weight", "offset": 21379768320, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 21458411520, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 21458432000, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 21641932800, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 22008934400, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 22008954880, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.31.attn_qkv.weight", "offset": 22061383680, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 22140026880, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 22140047360, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 22323548160, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 22690549760, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 22690570240, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.32.attn_qkv.weight", "offset": 22742999040, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.33.attn_norm.weight", "offset": 22821642240, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 22821662720, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 23005163520, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 23372165120, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 23372185600, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.33.attn_qkv.weight", "offset": 23424614400, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 23503257600, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 23503278080, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 23686778880, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 24053780480, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 24053800960, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.34.attn_qkv.weight", "offset": 24106229760, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 24184872960, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.35.attn_qkv.weight", "offset": 24237301760, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "output.weight", "offset": 24315944960, "shape": [ 5120, 32064 ], "size": 78643200, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 24644280320, "shape": [ 5120 ], "size": 328335360, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 24644300800, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 24827801600, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 25194803200, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 25194823680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 25194844160, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 25378344960, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 25745346560, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 25745367040, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.36.attn_qkv.weight", "offset": 25797795840, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 25876439040, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 25876459520, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 26059960320, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 26426961920, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 26426982400, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.37.attn_qkv.weight", "offset": 26479411200, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 26558054400, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 26558074880, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 26741575680, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 27108577280, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 27108597760, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.38.attn_qkv.weight", "offset": 27161026560, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 27239669760, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 27239690240, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 27423191040, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 27790192640, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 27790213120, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.39.attn_qkv.weight", "offset": 27842641920, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "output_norm.weight", "offset": 27921285120, "shape": [ 5120 ], "size": 78643200, "type": 0 } ], "version": 3 }