gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 11, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 22091776, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 44169216, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.0.post_attention_norm.weight", "offset": 66246656, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 66260992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 66275328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 66289664, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 69443584, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 75751424, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 82059264, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 85213184, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 85227520, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 107304960, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 129382400, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.1.post_attention_norm.weight", "offset": 151459840, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 151474176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 151488512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 151502848, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 154656768, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 160964608, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 167272448, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.2.attn_v.weight", "offset": 170426368, "shape": [ 3584, 2048 ], "size": 3153920, "type": 11 }, { "name": "blk.2.ffn_down.weight", "offset": 173580288, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 195657728, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 217735168, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.2.post_attention_norm.weight", "offset": 239812608, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 239826944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 239841280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 239855616, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 243009536, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 249317376, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 255625216, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 255639552, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.3.ffn_down.weight", "offset": 258793472, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 280870912, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 302948352, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.3.post_attention_norm.weight", "offset": 325025792, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 325040128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 325054464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 325068800, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 328222720, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 334530560, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 340838400, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 340852736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 340867072, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 362944512, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 385021952, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.4.post_attention_norm.weight", "offset": 407099392, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 407113728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 407128064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 407142400, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 410296320, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 416604160, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 422912000, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 426065920, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 426080256, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 448157696, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 470235136, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.5.post_attention_norm.weight", "offset": 492312576, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 492326912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 492341248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 492355584, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 495509504, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 501817344, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 508125184, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 511279104, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 511293440, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 533370880, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 555448320, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.6.post_attention_norm.weight", "offset": 577525760, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 577540096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 577554432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 577568768, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 580722688, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 587030528, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 593338368, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 596492288, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 596506624, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_k.weight", "offset": 618584064, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 621737984, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 628045824, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 634353664, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 637507584, "shape": [ 3584, 14336 ], "size": 3153920, "type": 11 }, { "name": "blk.7.ffn_down.weight", "offset": 659585024, "shape": [ 14336, 3584 ], "size": 22077440, "type": 11 }, { "name": "blk.7.post_attention_norm.weight", "offset": 681662464, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 681676800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 681691136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 681705472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 681719808, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 703797248, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 725874688, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 747952128, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 747966464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 747980800, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 751134720, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 757442560, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 763750400, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.8.post_attention_norm.weight", "offset": 766904320, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 766918656, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_down.weight", "offset": 770072576, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 792150016, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 792164352, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 814241792, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.9.post_attention_norm.weight", "offset": 836319232, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 836333568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 836347904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 836362240, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 839516160, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 845824000, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 852131840, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 858439680, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.10.ffn_down.weight", "offset": 861593600, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 883671040, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 905748480, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 908902400, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 908916736, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.10.post_attention_norm.weight", "offset": 930994176, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 931008512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 931022848, "shape": [ 4096, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 937330688, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 937345024, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 937359360, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 937373696, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 943681536, "shape": [ 3584, 14336 ], "size": 6307840, "type": 11 }, { "name": "blk.11.attn_k.weight", "offset": 965758976, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 968912896, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 968927232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 968941568, "shape": [ 4096, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.attn_v.weight", "offset": 975249408, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.11.ffn_down.weight", "offset": 978403328, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 1000480768, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1022558208, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1022572544, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.attn_v.weight", "offset": 1044649984, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 1047803904, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1054111744, "shape": [ 4096, 3584 ], "size": 6307840, "type": 11 }, { "name": "blk.12.attn_k.weight", "offset": 1060419584, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1063573504, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1063587840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1063602176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1063616512, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 1085693952, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1107771392, "shape": [ 4096, 3584 ], "size": 22077440, "type": 11 }, { "name": "blk.13.attn_v.weight", "offset": 1114079232, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1117233152, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1117247488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1117261824, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1117276160, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 1139353600, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.ffn_down.weight", "offset": 1161431040, "shape": [ 14336, 3584 ], "size": 22077440, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 1183508480, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1183522816, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 1186676736, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1192984576, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1192998912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1193013248, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 1215090688, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1237168128, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1259245568, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1259259904, "shape": [ 4096, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 1265567744, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1265582080, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1268736000, "shape": [ 3584, 2048 ], "size": 3153920, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 1271889920, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1278197760, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 1281351680, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 1287659520, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1287673856, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 1309751296, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1331828736, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1353906176, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1353920512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1353934848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1353949184, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1357103104, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 1363410944, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1363425280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1363439616, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1366593536, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1366607872, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1372915712, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 1376069632, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1382377472, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1382391808, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1404469248, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.16.ffn_down.weight", "offset": 1426546688, "shape": [ 14336, 3584 ], "size": 22077440, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 1448624128, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1448638464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1448652800, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 1451806720, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 1458114560, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 1458128896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1458143232, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1480220672, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.17.ffn_down.weight", "offset": 1502298112, "shape": [ 14336, 3584 ], "size": 22077440, "type": 11 }, { "name": "blk.17.attn_v.weight", "offset": 1524375552, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 1527529472, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1533837312, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 1536991232, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 1537005568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1537019904, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 1559097344, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1581174784, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.18.post_attention_norm.weight", "offset": 1603252224, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1603266560, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 1609574400, "shape": [ 4096, 3584 ], "size": 6307840, "type": 11 }, { "name": "blk.18.attn_k.weight", "offset": 1615882240, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 1619036160, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 1619050496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 1619064832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1619079168, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 1641156608, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 1663234048, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.19.post_attention_norm.weight", "offset": 1685311488, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 1685325824, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1685340160, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 1688494080, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 1694801920, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 1701109760, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.20.attn_k.weight", "offset": 1704263680, "shape": [ 3584, 2048 ], "size": 3153920, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 1707417600, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 1713725440, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 1720033280, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 1723187200, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1723201536, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 1745278976, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 1767356416, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.20.post_attention_norm.weight", "offset": 1789433856, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 1789448192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 1789462528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1789476864, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 1811554304, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 1811568640, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 1833646080, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.21.post_attention_norm.weight", "offset": 1855723520, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 1855737856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 1855752192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 1855766528, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 1858920448, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 1865228288, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 1871536128, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 1874690048, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1874704384, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 1896781824, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 1918859264, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.22.post_attention_norm.weight", "offset": 1940936704, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 1940951040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 1940965376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 1940979712, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 1944133632, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 1950441472, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 1956749312, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 1959903232, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1959917568, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 1981995008, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2004072448, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2026149888, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2026164224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2026178560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2026192896, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 2029346816, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 2035654656, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2041962496, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 2045116416, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2045130752, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 2067208192, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2089285632, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2111363072, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2111377408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2111391744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2111406080, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 2114560000, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 2120867840, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2127175680, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 2130329600, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2130343936, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 2152421376, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2174498816, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.25.post_attention_norm.weight", "offset": 2196576256, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 2196590592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2196604928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2196619264, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 2199773184, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 2206081024, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2212388864, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 2215542784, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2215557120, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 2237634560, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2259712000, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.26.post_attention_norm.weight", "offset": 2281789440, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 2281803776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2281818112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2281832448, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 2284986368, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 2291294208, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2297602048, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 2300755968, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2300770304, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 2322847744, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2344925184, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.27.post_attention_norm.weight", "offset": 2367002624, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 2367016960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 2367031296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 2367045632, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 2370199552, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.27.attn_q.weight", "offset": 2376507392, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2382815232, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 2385969152, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2385983488, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.28.ffn_gate.weight", "offset": 2408060928, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2430138368, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.28.post_attention_norm.weight", "offset": 2452215808, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 2452230144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 2452244480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 2452258816, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 2455412736, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.28.attn_q.weight", "offset": 2461720576, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 2468028416, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 2471182336, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 2471196672, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.29.ffn_gate.weight", "offset": 2493274112, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 2515351552, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.29.post_attention_norm.weight", "offset": 2537428992, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 2537443328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 2537457664, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 2537472000, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 2540625920, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.29.attn_q.weight", "offset": 2546933760, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 2553241600, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 2556395520, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2556409856, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.30.ffn_gate.weight", "offset": 2578487296, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 2600564736, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.30.post_attention_norm.weight", "offset": 2622642176, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 2622656512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 2622670848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 2622685184, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 2625839104, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.30.attn_q.weight", "offset": 2632146944, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 2638454784, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.31.attn_k.weight", "offset": 2641608704, "shape": [ 3584, 2048 ], "size": 3153920, "type": 11 }, { "name": "blk.31.ffn_down.weight", "offset": 2644762624, "shape": [ 14336, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.31.ffn_gate.weight", "offset": 2666840064, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 2688917504, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.31.post_attention_norm.weight", "offset": 2710994944, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 2711009280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 2711023616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 2711037952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 2711052288, "shape": [ 4096, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.31.attn_q.weight", "offset": 2717360128, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 2723667968, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.32.ffn_gate.weight", "offset": 2726821888, "shape": [ 3584, 14336 ], "size": 3153920, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 2748899328, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_k.weight", "offset": 2770976768, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 2774130688, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.32.attn_q.weight", "offset": 2780438528, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 2786746368, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 2789900288, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 2789914624, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.32.post_attention_norm.weight", "offset": 2811992064, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 2812006400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 2812020736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 2812035072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 2812049408, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.33.ffn_gate.weight", "offset": 2834126848, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 2856204288, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.33.post_attention_norm.weight", "offset": 2878281728, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 2878296064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 2878310400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 2878324736, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 2881478656, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.33.attn_q.weight", "offset": 2887786496, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 2894094336, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 2897248256, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 2897262592, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.34.ffn_gate.weight", "offset": 2919340032, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 2941417472, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.34.post_attention_norm.weight", "offset": 2963494912, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 2963509248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 2963523584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 2963537920, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 2966691840, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.34.attn_q.weight", "offset": 2972999680, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 2979307520, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 2982461440, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 2982475776, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.35.ffn_gate.weight", "offset": 3004553216, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 3026630656, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.35.post_attention_norm.weight", "offset": 3048708096, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 3048722432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 3048736768, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3048751104, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 3051905024, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.35.attn_q.weight", "offset": 3058212864, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 3064520704, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 3067674624, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3067688960, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.36.ffn_gate.weight", "offset": 3089766400, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 3111843840, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.36.post_attention_norm.weight", "offset": 3133921280, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 3133935616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 3133949952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3133964288, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 3137118208, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.36.attn_q.weight", "offset": 3143426048, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 3149733888, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 3152887808, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3152902144, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.37.ffn_gate.weight", "offset": 3174979584, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 3197057024, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.37.post_attention_norm.weight", "offset": 3219134464, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 3219148800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 3219163136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 3219177472, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 3222331392, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.37.attn_q.weight", "offset": 3228639232, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 3234947072, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 3238100992, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 3238115328, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.38.ffn_gate.weight", "offset": 3260192768, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 3282270208, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.38.post_attention_norm.weight", "offset": 3304347648, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 3304361984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 3304376320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 3304390656, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 3307544576, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.38.attn_q.weight", "offset": 3313852416, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 3320160256, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 3323314176, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 3323328512, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.39.ffn_gate.weight", "offset": 3345405952, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 3367483392, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.39.post_attention_norm.weight", "offset": 3389560832, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 3389575168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 3389589504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 3389603840, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 3392757760, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.39.attn_q.weight", "offset": 3399065600, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 3405373440, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 3408527360, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 3408541696, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.40.ffn_gate.weight", "offset": 3430619136, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 3452696576, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.40.post_attention_norm.weight", "offset": 3474774016, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 3474788352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 3474802688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 3474817024, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 3477970944, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.40.attn_q.weight", "offset": 3484278784, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 3490586624, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 3493740544, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 3493754880, "shape": [ 14336, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.41.ffn_gate.weight", "offset": 3515832320, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 3537909760, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.41.post_attention_norm.weight", "offset": 3559987200, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 3560001536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 3560015872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 3560030208, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 3563184128, "shape": [ 4096, 3584 ], "size": 3153920, "type": 11 }, { "name": "blk.41.attn_q.weight", "offset": 3569491968, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 3575799808, "shape": [ 3584, 2048 ], "size": 6307840, "type": 11 }, { "name": "output_norm.weight", "offset": 3578953728, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "token_embd.weight", "offset": 3578968064, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }