yi

{ "metadata": { "general.architecture": "llama", "general.file_type": 10, "general.name": "Yi-1.5-9B-Chat", "general.quantization_version": 2, "llama.attention.head_count": 32, "llama.attention.head_count_kv": 4, "llama.attention.layer_norm_rms_epsilon": 0.000001, "llama.block_count": 48, "llama.context_length": 4096, "llama.embedding_length": 4096, "llama.feed_forward_length": 11008, "llama.rope.dimension_count": 128, "llama.rope.freq_base": 5000000, "llama.vocab_size": 64000, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 2, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (64000 values)", "tokenizer.ggml.token_type": "... (64000 values)", "tokenizer.ggml.tokens": "... (64000 values)" }, "num_params": 8829407232, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 86016000, "shape": [ 4096 ], "size": 86016000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 86032384, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 105406464, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 120201216, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 134995968, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 135012352, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 135700480, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 142909440, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.0.attn_v.weight", "offset": 148414464, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 149594112, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 149610496, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 168984576, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 183779328, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 198574080, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 198590464, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 199278592, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 206487552, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.1.attn_v.weight", "offset": 211992576, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 213172224, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 213188608, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 232562688, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 247357440, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 262152192, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 262168576, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 262856704, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 270065664, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.10.attn_v.weight", "offset": 275570688, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 276750336, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 276766720, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 296140800, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 310935552, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 325730304, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 325746688, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 326434816, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 333643776, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.11.attn_v.weight", "offset": 339148800, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 340328448, "shape": [ 4096, 11008 ], "size": 1179648, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 355123200, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.12.attn_k.weight", "offset": 369917952, "shape": [ 4096, 512 ], "size": 14794752, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 370606080, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 377815040, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.12.attn_v.weight", "offset": 383320064, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 384499712, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 384516096, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 403890176, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 418684928, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 433479680, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 433496064, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 434184192, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 441393152, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.2.attn_v.weight", "offset": 446898176, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 448077824, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 448094208, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 467468288, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 482263040, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 497057792, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 497074176, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 497762304, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 504971264, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.3.attn_v.weight", "offset": 510476288, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 511655936, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 511672320, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 531046400, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 545841152, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 560635904, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 560652288, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 561340416, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 568549376, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.4.attn_v.weight", "offset": 574054400, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 575234048, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 575250432, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 594624512, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 609419264, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 624214016, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 624230400, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 624918528, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 632127488, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.5.attn_v.weight", "offset": 637632512, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 638812160, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 638828544, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 658202624, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 672997376, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 687792128, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 687808512, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 688496640, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 695705600, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.6.attn_v.weight", "offset": 701210624, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 702390272, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 702406656, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 721780736, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.7.ffn_up.weight", "offset": 736575488, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 751370240, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 751386624, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 752074752, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 759283712, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.7.attn_v.weight", "offset": 764788736, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 765968384, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 765984768, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 785358848, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 800153600, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 814948352, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 814964736, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 815652864, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 822861824, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.8.attn_v.weight", "offset": 828366848, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 829546496, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 829562880, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 848936960, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 863731712, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 878526464, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 878542848, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 879230976, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 886439936, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.9.attn_v.weight", "offset": 891944960, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 893124608, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 893140992, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 912515072, "shape": [ 4096 ], "size": 19374080, "type": 0 }, { "name": "blk.13.attn_norm.weight", "offset": 912531456, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 912547840, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 931921920, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 946716672, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 961511424, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 961527808, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 962215936, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 969424896, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.13.attn_v.weight", "offset": 974929920, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 976109568, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 976125952, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 995500032, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.14.ffn_up.weight", "offset": 1010294784, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 1025089536, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1025105920, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 1025794048, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 1033003008, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.14.attn_v.weight", "offset": 1038508032, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1039687680, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1039704064, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 1059078144, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 1073872896, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 1088667648, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1088684032, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 1089372160, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 1096581120, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.15.attn_v.weight", "offset": 1102086144, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 1103265792, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1103282176, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1122656256, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.16.ffn_up.weight", "offset": 1137451008, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 1152245760, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1152262144, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 1152950272, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 1160159232, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.16.attn_v.weight", "offset": 1165664256, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1166843904, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1166860288, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1186234368, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 1201029120, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 1215823872, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1215840256, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 1216528384, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 1223737344, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.17.attn_v.weight", "offset": 1229242368, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 1230422016, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1230438400, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 1249812480, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.18.ffn_up.weight", "offset": 1264607232, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 1279401984, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1279418368, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 1280106496, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 1287315456, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.18.attn_v.weight", "offset": 1292820480, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 1294000128, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1294016512, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 1313390592, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.19.ffn_up.weight", "offset": 1328185344, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 1342980096, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1342996480, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 1343684608, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 1350893568, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.19.attn_v.weight", "offset": 1356398592, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 1357578240, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1357594624, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 1376968704, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 1391763456, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 1406558208, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 1406574592, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 1407262720, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 1414471680, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.20.attn_v.weight", "offset": 1419976704, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 1421156352, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1421172736, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 1440546816, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 1455341568, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 1470136320, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 1470152704, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 1470840832, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 1478049792, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.21.attn_v.weight", "offset": 1483554816, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 1484734464, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1484750848, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 1504124928, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 1518919680, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 1533714432, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 1533730816, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 1534418944, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 1541627904, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.22.attn_v.weight", "offset": 1547132928, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 1548312576, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1548328960, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 1567703040, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.23.ffn_up.weight", "offset": 1582497792, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 1597292544, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 1597308928, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 1597997056, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 1605206016, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.23.attn_v.weight", "offset": 1610711040, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 1611890688, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 1611907072, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 1631281152, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 1646075904, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 1660870656, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 1660887040, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 1661575168, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 1668784128, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.24.attn_v.weight", "offset": 1674289152, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 1675468800, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 1675485184, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 1694859264, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.25.ffn_up.weight", "offset": 1709654016, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 1724448768, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 1724465152, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 1725153280, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 1732362240, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.25.attn_v.weight", "offset": 1737867264, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 1739046912, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 1739063296, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 1758437376, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 1773232128, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 1788026880, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 1788043264, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 1788731392, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 1795940352, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.26.attn_v.weight", "offset": 1801445376, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.27.attn_k.weight", "offset": 1802625024, "shape": [ 4096, 512 ], "size": 1179648, "type": 10 }, { "name": "blk.27.attn_q.weight", "offset": 1803313152, "shape": [ 4096, 4096 ], "size": 688128, "type": 10 }, { "name": "blk.27.attn_v.weight", "offset": 1808818176, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 1809997824, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 1810014208, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 1829388288, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 1844183040, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 1858977792, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 1858994176, "shape": [ 4096, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 1866203136, "shape": [ 4096 ], "size": 7208960, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 1866219520, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 1885593600, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.28.ffn_up.weight", "offset": 1900388352, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 1915183104, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 1915199488, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 1915887616, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 1923096576, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.28.attn_v.weight", "offset": 1928601600, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 1929781248, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 1929797632, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 1949171712, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.29.ffn_up.weight", "offset": 1963966464, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 1978761216, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 1978777600, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 1979465728, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 1986674688, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.29.attn_v.weight", "offset": 1992179712, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 1993359360, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 1993375744, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 2012749824, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.30.ffn_up.weight", "offset": 2027544576, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 2042339328, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 2042355712, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 2043043840, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 2050252800, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.30.attn_v.weight", "offset": 2055757824, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 2056937472, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 2056953856, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 2076327936, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.31.ffn_up.weight", "offset": 2091122688, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 2105917440, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.31.attn_k.weight", "offset": 2105933824, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.31.attn_output.weight", "offset": 2106621952, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 2113830912, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.31.attn_v.weight", "offset": 2119335936, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 2120515584, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 2120531968, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 2139906048, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.32.ffn_up.weight", "offset": 2154700800, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 2169495552, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.32.attn_k.weight", "offset": 2169511936, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 2170200064, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 2177409024, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.32.attn_v.weight", "offset": 2182914048, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 2184093696, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 2184110080, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 2203484160, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.33.ffn_up.weight", "offset": 2218278912, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 2233073664, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 2233090048, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 2233778176, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 2240987136, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.33.attn_v.weight", "offset": 2246492160, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 2247671808, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 2247688192, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 2267062272, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.34.ffn_up.weight", "offset": 2281857024, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 2296651776, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 2296668160, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 2297356288, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 2304565248, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.34.attn_v.weight", "offset": 2310070272, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 2311249920, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 2311266304, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 2330640384, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.35.ffn_up.weight", "offset": 2345435136, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 2360229888, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 2360246272, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 2360934400, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 2368143360, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.35.attn_v.weight", "offset": 2373648384, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 2374828032, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 2374844416, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 2394218496, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.36.ffn_up.weight", "offset": 2409013248, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.36.ffn_norm.weight", "offset": 2423808000, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 2423824384, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 2424512512, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 2431721472, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.36.attn_v.weight", "offset": 2437226496, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 2438406144, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 2438422528, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 2457796608, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.37.ffn_up.weight", "offset": 2472591360, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 2487386112, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 2487402496, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 2488090624, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 2495299584, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.37.attn_v.weight", "offset": 2500804608, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 2501984256, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 2502000640, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 2521374720, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.38.ffn_up.weight", "offset": 2536169472, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 2550964224, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 2550980608, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 2551668736, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 2558877696, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.38.attn_v.weight", "offset": 2564382720, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 2565562368, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 2565578752, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 2584952832, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.39.ffn_up.weight", "offset": 2599747584, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 2614542336, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 2614558720, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 2615246848, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 2622455808, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.39.attn_v.weight", "offset": 2627960832, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 2629140480, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 2629156864, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.40.ffn_gate.weight", "offset": 2648530944, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.40.ffn_up.weight", "offset": 2663325696, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.40.ffn_norm.weight", "offset": 2678120448, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 2678136832, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.40.attn_output.weight", "offset": 2678824960, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 2686033920, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.40.attn_v.weight", "offset": 2691538944, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 2692718592, "shape": [ 4096, 11008 ], "size": 1179648, "type": 10 }, { "name": "blk.41.attn_k.weight", "offset": 2707513344, "shape": [ 4096, 512 ], "size": 14794752, "type": 10 }, { "name": "blk.41.attn_output.weight", "offset": 2708201472, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 2715410432, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.41.attn_v.weight", "offset": 2720915456, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "output.weight", "offset": 2722095104, "shape": [ 4096, 64000 ], "size": 1179648, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 2937135104, "shape": [ 4096 ], "size": 215040000, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 2937151488, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 2956525568, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.41.ffn_norm.weight", "offset": 2971320320, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.42.attn_norm.weight", "offset": 2971336704, "shape": [ 4096 ], "size": 16384, "type": 0 }, { "name": "blk.42.ffn_down.weight", "offset": 2971353088, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.42.ffn_gate.weight", "offset": 2990727168, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.42.ffn_up.weight", "offset": 3005521920, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.42.ffn_norm.weight", "offset": 3020316672, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.42.attn_k.weight", "offset": 3020333056, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.42.attn_output.weight", "offset": 3021021184, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.42.attn_q.weight", "offset": 3028230144, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.42.attn_v.weight", "offset": 3033735168, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 3034914816, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.43.ffn_down.weight", "offset": 3034931200, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.43.ffn_gate.weight", "offset": 3054305280, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.43.ffn_up.weight", "offset": 3069100032, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.43.ffn_norm.weight", "offset": 3083894784, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.43.attn_k.weight", "offset": 3083911168, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.43.attn_output.weight", "offset": 3084599296, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.43.attn_q.weight", "offset": 3091808256, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.43.attn_v.weight", "offset": 3097313280, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.44.attn_norm.weight", "offset": 3098492928, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.44.ffn_down.weight", "offset": 3098509312, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.44.ffn_gate.weight", "offset": 3117883392, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.44.ffn_up.weight", "offset": 3132678144, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.44.ffn_norm.weight", "offset": 3147472896, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.44.attn_k.weight", "offset": 3147489280, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.44.attn_output.weight", "offset": 3148177408, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.44.attn_q.weight", "offset": 3155386368, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.44.attn_v.weight", "offset": 3160891392, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 3162071040, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.45.ffn_down.weight", "offset": 3162087424, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.45.ffn_gate.weight", "offset": 3181461504, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.45.ffn_up.weight", "offset": 3196256256, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.45.ffn_norm.weight", "offset": 3211051008, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.45.attn_k.weight", "offset": 3211067392, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.45.attn_output.weight", "offset": 3211755520, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.45.attn_q.weight", "offset": 3218964480, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.45.attn_v.weight", "offset": 3224469504, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 3225649152, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.46.ffn_down.weight", "offset": 3225665536, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.46.ffn_gate.weight", "offset": 3245039616, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.46.ffn_up.weight", "offset": 3259834368, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.46.ffn_norm.weight", "offset": 3274629120, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.46.attn_k.weight", "offset": 3274645504, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.46.attn_output.weight", "offset": 3275333632, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.46.attn_q.weight", "offset": 3282542592, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.46.attn_v.weight", "offset": 3288047616, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 3289227264, "shape": [ 4096 ], "size": 1179648, "type": 0 }, { "name": "blk.47.ffn_down.weight", "offset": 3289243648, "shape": [ 11008, 4096 ], "size": 16384, "type": 11 }, { "name": "blk.47.ffn_gate.weight", "offset": 3308617728, "shape": [ 4096, 11008 ], "size": 19374080, "type": 10 }, { "name": "blk.47.ffn_up.weight", "offset": 3323412480, "shape": [ 4096, 11008 ], "size": 14794752, "type": 10 }, { "name": "blk.47.ffn_norm.weight", "offset": 3338207232, "shape": [ 4096 ], "size": 14794752, "type": 0 }, { "name": "blk.47.attn_k.weight", "offset": 3338223616, "shape": [ 4096, 512 ], "size": 16384, "type": 10 }, { "name": "blk.47.attn_output.weight", "offset": 3338911744, "shape": [ 4096, 4096 ], "size": 688128, "type": 11 }, { "name": "blk.47.attn_q.weight", "offset": 3346120704, "shape": [ 4096, 4096 ], "size": 7208960, "type": 10 }, { "name": "blk.47.attn_v.weight", "offset": 3351625728, "shape": [ 4096, 512 ], "size": 5505024, "type": 12 }, { "name": "output_norm.weight", "offset": 3352805376, "shape": [ 4096 ], "size": 1179648, "type": 0 } ], "version": 3 }