phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 11, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 70540800, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561280, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 109985280, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 188833280, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 188853760, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.attn_qkv.weight", "offset": 200117760, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 217013760, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 217034240, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 256458240, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 335306240, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 335326720, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.attn_qkv.weight", "offset": 346590720, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 363486720, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 363507200, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 402931200, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 481779200, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 481799680, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.attn_qkv.weight", "offset": 493063680, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 509959680, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 509980160, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 549404160, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 628252160, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 628272640, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.attn_qkv.weight", "offset": 639536640, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 656432640, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 656453120, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 695877120, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 774725120, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 774745600, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.attn_qkv.weight", "offset": 786009600, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 802905600, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 802926080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 842350080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 921198080, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 921218560, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.attn_qkv.weight", "offset": 932482560, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 949378560, "shape": [ 5120, 35840 ], "size": 16896000, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1028226560, "shape": [ 5120, 5120 ], "size": 78848000, "type": 11 }, { "name": "blk.6.attn_qkv.weight", "offset": 1039490560, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 1056386560, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1056407040, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1095831040, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1174679040, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1174699520, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.attn_qkv.weight", "offset": 1185963520, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 1202859520, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1202880000, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1242304000, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1321152000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1321172480, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.attn_qkv.weight", "offset": 1332436480, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1349332480, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1349352960, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1388776960, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1467624960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1467645440, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.attn_qkv.weight", "offset": 1478909440, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 1495805440, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1495825920, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1535249920, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1614097920, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1614118400, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.attn_qkv.weight", "offset": 1625382400, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1642278400, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1642298880, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1681722880, "shape": [ 5120 ], "size": 39424000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1681743360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1681763840, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1721187840, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1800035840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1800056320, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.attn_qkv.weight", "offset": 1811320320, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1828216320, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1828236800, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1867660800, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1946508800, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1946529280, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.attn_qkv.weight", "offset": 1957793280, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1974689280, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1974709760, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2014133760, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2092981760, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2093002240, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.attn_qkv.weight", "offset": 2104266240, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 2121162240, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2121182720, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 2160606720, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2239454720, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2239475200, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.attn_qkv.weight", "offset": 2250739200, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 2267635200, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2267655680, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 2307079680, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2385927680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2385948160, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.attn_qkv.weight", "offset": 2397212160, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 2414108160, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2414128640, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2453552640, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2532400640, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2532421120, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.attn_qkv.weight", "offset": 2543685120, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 2560581120, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2560601600, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2600025600, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 2678873600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2678894080, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.attn_qkv.weight", "offset": 2690158080, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 2707054080, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2707074560, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2746498560, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2825346560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2825367040, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.attn_qkv.weight", "offset": 2836631040, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 2853527040, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2853547520, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2892971520, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2971819520, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2971840000, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.attn_qkv.weight", "offset": 2983104000, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 3000000000, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3000020480, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 3039444480, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3118292480, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3118312960, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.attn_qkv.weight", "offset": 3129576960, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 3146472960, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.21.attn_qkv.weight", "offset": 3157736960, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 3174632960, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3174653440, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 3214077440, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 3292925440, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 3292945920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3292966400, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 3332390400, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3411238400, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 3411258880, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.attn_qkv.weight", "offset": 3422522880, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 3439418880, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3439439360, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3478863360, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3557711360, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 3557731840, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.attn_qkv.weight", "offset": 3568995840, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 3585891840, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3585912320, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3625336320, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3704184320, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 3704204800, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.attn_qkv.weight", "offset": 3715468800, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 3732364800, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3732385280, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3771809280, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3850657280, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 3850677760, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.attn_qkv.weight", "offset": 3861941760, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 3878837760, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3878858240, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3918282240, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3997130240, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 3997150720, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.attn_qkv.weight", "offset": 4008414720, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 4025310720, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4025331200, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 4064755200, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4143603200, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 4143623680, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.attn_qkv.weight", "offset": 4154887680, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 4171783680, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.28.attn_qkv.weight", "offset": 4183047680, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 4199943680, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4199964160, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 4239388160, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 4318236160, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 4318256640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4318277120, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 4357701120, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 4436549120, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 4436569600, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.attn_qkv.weight", "offset": 4447833600, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 4464729600, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4464750080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 4504174080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 4583022080, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 4583042560, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.attn_qkv.weight", "offset": 4594306560, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 4611202560, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4611223040, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 4650647040, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 4729495040, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4729515520, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.attn_qkv.weight", "offset": 4740779520, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 4757675520, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4757696000, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 4797120000, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 4875968000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 4875988480, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.attn_qkv.weight", "offset": 4887252480, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 4904148480, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4904168960, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 4943592960, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 5022440960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 5022461440, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.attn_qkv.weight", "offset": 5033725440, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 5050621440, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5050641920, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 5090065920, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 5168913920, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 5168934400, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.attn_qkv.weight", "offset": 5180198400, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 5197094400, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.35.attn_qkv.weight", "offset": 5208358400, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "output.weight", "offset": 5225254400, "shape": [ 5120, 32064 ], "size": 16896000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 5359923200, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5359943680, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 5399367680, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 5478215680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5478236160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5478256640, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 5517680640, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 5596528640, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 5596549120, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.attn_qkv.weight", "offset": 5607813120, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 5624709120, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5624729600, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 5664153600, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 5743001600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 5743022080, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.attn_qkv.weight", "offset": 5754286080, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 5771182080, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5771202560, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 5810626560, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 5889474560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 5889495040, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.attn_qkv.weight", "offset": 5900759040, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 5917655040, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5917675520, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 5957099520, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 6035947520, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 6035968000, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.attn_qkv.weight", "offset": 6047232000, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "output_norm.weight", "offset": 6064128000, "shape": [ 5120 ], "size": 16896000, "type": 0 } ], "version": 3 }