phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 1, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 328335872, "shape": [ 5120 ], "size": 328335360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 328356352, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 511857152, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 878858752, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 878879232, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.0.attn_qkv.weight", "offset": 931308032, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 1009951232, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 1009971712, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 1193472512, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 1560474112, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 1560494592, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.1.attn_qkv.weight", "offset": 1612923392, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 1691566592, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1691587072, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 1875087872, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 2242089472, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 2242109952, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.2.attn_qkv.weight", "offset": 2294538752, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 2373181952, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 2373202432, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 2556703232, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 2923704832, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 2923725312, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.3.attn_qkv.weight", "offset": 2976154112, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3054797312, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3054817792, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3238318592, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 3605320192, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 3605340672, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.4.attn_qkv.weight", "offset": 3657769472, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 3736412672, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3736433152, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 3919933952, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 4286935552, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 4286956032, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.5.attn_qkv.weight", "offset": 4339384832, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 4418028032, "shape": [ 5120, 35840 ], "size": 78643200, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 4785029632, "shape": [ 5120, 5120 ], "size": 367001600, "type": 1 }, { "name": "blk.6.attn_qkv.weight", "offset": 4837458432, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 4916101632, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 4916122112, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 5099622912, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 5466624512, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 5466644992, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.10.attn_qkv.weight", "offset": 5519073792, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 5597716992, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 5597737472, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 5781238272, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 6148239872, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 6148260352, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.11.attn_qkv.weight", "offset": 6200689152, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 6279332352, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 6279352832, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 6462853632, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 6829855232, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 6829875712, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.12.attn_qkv.weight", "offset": 6882304512, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 6960947712, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 6960968192, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 7144468992, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 7511470592, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 7511491072, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.13.attn_qkv.weight", "offset": 7563919872, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 7642563072, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 7642583552, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 7826084352, "shape": [ 5120 ], "size": 183500800, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 7826104832, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 7826125312, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 8009626112, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 8376627712, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 8376648192, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.7.attn_qkv.weight", "offset": 8429076992, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 8507720192, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 8507740672, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 8691241472, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 9058243072, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 9058263552, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.8.attn_qkv.weight", "offset": 9110692352, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 9189335552, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 9189356032, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 9372856832, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 9739858432, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 9739878912, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.9.attn_qkv.weight", "offset": 9792307712, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 9870950912, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 9870971392, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 10054472192, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 10421473792, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 10421494272, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.14.attn_qkv.weight", "offset": 10473923072, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 10552566272, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 10552586752, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 10736087552, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 11103089152, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 11103109632, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.15.attn_qkv.weight", "offset": 11155538432, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 11234181632, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 11234202112, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 11417702912, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 11784704512, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 11784724992, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.16.attn_qkv.weight", "offset": 11837153792, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 11915796992, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 11915817472, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 12099318272, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 12466319872, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 12466340352, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.17.attn_qkv.weight", "offset": 12518769152, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 12597412352, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 12597432832, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 12780933632, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 13147935232, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 13147955712, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.18.attn_qkv.weight", "offset": 13200384512, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 13279027712, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 13279048192, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 13462548992, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "output.weight\u0002\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000", "offset": 13829550592, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 13829571072, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.19.attn_qkv.weight", "offset": 13881999872, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 13960643072, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 13960663552, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 14144164352, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 14511165952, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 14511186432, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.20.attn_qkv.weight", "offset": 14563615232, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 14642258432, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.21.attn_qkv.weight", "offset": 14694687232, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 14773330432, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 14773350912, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 14956851712, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 15323853312, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 15323873792, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 15323894272, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 15507395072, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 15874396672, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 15874417152, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.22.attn_qkv.weight", "offset": 15926845952, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 16005489152, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 16005509632, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 16189010432, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 16556012032, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 16556032512, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.23.attn_qkv.weight", "offset": 16608461312, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 16687104512, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 16687124992, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 16870625792, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 17237627392, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 17237647872, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.24.attn_qkv.weight", "offset": 17290076672, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 17368719872, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 17368740352, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 17552241152, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 17919242752, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 17919263232, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.25.attn_qkv.weight", "offset": 17971692032, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 18050335232, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 18050355712, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 18233856512, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 18600858112, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 18600878592, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.26.attn_qkv.weight", "offset": 18653307392, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 18731950592, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 18731971072, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 18915471872, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 19282473472, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 19282493952, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.27.attn_qkv.weight", "offset": 19334922752, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 19413565952, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.28.attn_qkv.weight", "offset": 19465994752, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 19544637952, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 19544658432, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 19728159232, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 20095160832, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 20095181312, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 20095201792, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 20278702592, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 20645704192, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 20645724672, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.29.attn_qkv.weight", "offset": 20698153472, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 20776796672, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 20776817152, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 20960317952, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 21327319552, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 21327340032, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.30.attn_qkv.weight", "offset": 21379768832, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 21458412032, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 21458432512, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 21641933312, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 22008934912, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 22008955392, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.31.attn_qkv.weight", "offset": 22061384192, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 22140027392, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 22140047872, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.32.ffn_up.weight", "offset": 22323548672, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 22690550272, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 22690570752, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.32.attn_qkv.weight", "offset": 22742999552, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.33.attn_norm.weight", "offset": 22821642752, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 22821663232, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.33.ffn_up.weight", "offset": 23005164032, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 23372165632, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 23372186112, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.33.attn_qkv.weight", "offset": 23424614912, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 23503258112, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 23503278592, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.34.ffn_up.weight", "offset": 23686779392, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 24053780992, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 24053801472, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.34.attn_qkv.weight", "offset": 24106230272, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 24184873472, "shape": [ 5120, 5120 ], "size": 78643200, "type": 1 }, { "name": "blk.35.attn_qkv.weight", "offset": 24237302272, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "output.weight", "offset": 24315945472, "shape": [ 5120, 32064 ], "size": 78643200, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 24644280832, "shape": [ 5120 ], "size": 328335360, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 24644301312, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.35.ffn_up.weight", "offset": 24827802112, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 25194803712, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 25194824192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 25194844672, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.36.ffn_up.weight", "offset": 25378345472, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 25745347072, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 25745367552, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.36.attn_qkv.weight", "offset": 25797796352, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 25876439552, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 25876460032, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.37.ffn_up.weight", "offset": 26059960832, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 26426962432, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 26426982912, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.37.attn_qkv.weight", "offset": 26479411712, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 26558054912, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 26558075392, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.38.ffn_up.weight", "offset": 26741576192, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 27108577792, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 27108598272, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.38.attn_qkv.weight", "offset": 27161027072, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 27239670272, "shape": [ 5120 ], "size": 78643200, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 27239690752, "shape": [ 17920, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.39.ffn_up.weight", "offset": 27423191552, "shape": [ 5120, 35840 ], "size": 183500800, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 27790193152, "shape": [ 5120 ], "size": 367001600, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 27790213632, "shape": [ 5120, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.39.attn_qkv.weight", "offset": 27842642432, "shape": [ 5120, 7680 ], "size": 52428800, "type": 1 }, { "name": "output_norm.weight", "offset": 27921285632, "shape": [ 5120 ], "size": 78643200, "type": 0 } ], "version": 3 }