phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 3, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 102604800, "shape": [ 5120 ], "size": 102604800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 102625280, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 159969280, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 274657280, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 274677760, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.0.attn_qkv.weight", "offset": 291061760, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 315637760, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 315658240, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 373002240, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 487690240, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 487710720, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.1.attn_qkv.weight", "offset": 504094720, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 528670720, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 528691200, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 586035200, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 700723200, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 700743680, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.2.attn_qkv.weight", "offset": 717127680, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 741703680, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 741724160, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 799068160, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 913756160, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 913776640, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.3.attn_qkv.weight", "offset": 930160640, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 954736640, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 954757120, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 1012101120, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 1126789120, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1126809600, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.4.attn_qkv.weight", "offset": 1143193600, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 1167769600, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1167790080, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 1225134080, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 1339822080, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1339842560, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.5.attn_qkv.weight", "offset": 1356226560, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 1380802560, "shape": [ 5120, 35840 ], "size": 24576000, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 1495490560, "shape": [ 5120, 5120 ], "size": 114688000, "type": 3 }, { "name": "blk.6.attn_qkv.weight", "offset": 1511874560, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 1536450560, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1536471040, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 1593815040, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 1708503040, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1708523520, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.10.attn_qkv.weight", "offset": 1724907520, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 1749483520, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1749504000, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 1806848000, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 1921536000, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1921556480, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.11.attn_qkv.weight", "offset": 1937940480, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 1962516480, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1962536960, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 2019880960, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 2134568960, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2134589440, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.12.attn_qkv.weight", "offset": 2150973440, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 2175549440, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2175569920, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 2232913920, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 2347601920, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2347622400, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.13.attn_qkv.weight", "offset": 2364006400, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 2388582400, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2388602880, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 2445946880, "shape": [ 5120 ], "size": 57344000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2445967360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2445987840, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 2503331840, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 2618019840, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2618040320, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.7.attn_qkv.weight", "offset": 2634424320, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 2659000320, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2659020800, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 2716364800, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 2831052800, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2831073280, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.8.attn_qkv.weight", "offset": 2847457280, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 2872033280, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2872053760, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 2929397760, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 3044085760, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3044106240, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.9.attn_qkv.weight", "offset": 3060490240, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 3085066240, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3085086720, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 3142430720, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 3257118720, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3257139200, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.14.attn_qkv.weight", "offset": 3273523200, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 3298099200, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3298119680, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 3355463680, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 3470151680, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3470172160, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.15.attn_qkv.weight", "offset": 3486556160, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 3511132160, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3511152640, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 3568496640, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 3683184640, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3683205120, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.16.attn_qkv.weight", "offset": 3699589120, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 3724165120, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3724185600, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 3781529600, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 3896217600, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3896238080, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.17.attn_qkv.weight", "offset": 3912622080, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 3937198080, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3937218560, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 3994562560, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 4109250560, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4109271040, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.18.attn_qkv.weight", "offset": 4125655040, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 4150231040, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4150251520, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 4207595520, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 4322283520, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4322304000, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.19.attn_qkv.weight", "offset": 4338688000, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 4363264000, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4363284480, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 4420628480, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 4535316480, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4535336960, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.20.attn_qkv.weight", "offset": 4551720960, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 4576296960, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.21.attn_qkv.weight", "offset": 4592680960, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 4617256960, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4617277440, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 4674621440, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 4789309440, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4789329920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4789350400, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 4846694400, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 4961382400, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4961402880, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.22.attn_qkv.weight", "offset": 4977786880, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 5002362880, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5002383360, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 5059727360, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 5174415360, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5174435840, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.23.attn_qkv.weight", "offset": 5190819840, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 5215395840, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5215416320, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 5272760320, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 5387448320, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5387468800, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.24.attn_qkv.weight", "offset": 5403852800, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 5428428800, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5428449280, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 5485793280, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 5600481280, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5600501760, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.25.attn_qkv.weight", "offset": 5616885760, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 5641461760, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5641482240, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 5698826240, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 5813514240, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5813534720, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.26.attn_qkv.weight", "offset": 5829918720, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 5854494720, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5854515200, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 5911859200, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 6026547200, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6026567680, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.27.attn_qkv.weight", "offset": 6042951680, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 6067527680, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.28.attn_qkv.weight", "offset": 6083911680, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 6108487680, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6108508160, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 6165852160, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 6280540160, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6280560640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6280581120, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 6337925120, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 6452613120, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 6452633600, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.29.attn_qkv.weight", "offset": 6469017600, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 6493593600, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6493614080, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 6550958080, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 6665646080, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6665666560, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.30.attn_qkv.weight", "offset": 6682050560, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.31.attn_norm.weight", "offset": 6706626560, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6706647040, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 6763991040, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 6878679040, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6878699520, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.31.attn_qkv.weight", "offset": 6895083520, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 6919659520, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6919680000, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.32.ffn_up.weight", "offset": 6977024000, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.32.ffn_norm.weight", "offset": 7091712000, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 7091732480, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.32.attn_qkv.weight", "offset": 7108116480, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.33.attn_norm.weight", "offset": 7132692480, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7132712960, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.33.ffn_up.weight", "offset": 7190056960, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.33.ffn_norm.weight", "offset": 7304744960, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 7304765440, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.33.attn_qkv.weight", "offset": 7321149440, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 7345725440, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7345745920, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.34.ffn_up.weight", "offset": 7403089920, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.34.ffn_norm.weight", "offset": 7517777920, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 7517798400, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.34.attn_qkv.weight", "offset": 7534182400, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 7558758400, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.35.attn_qkv.weight", "offset": 7575142400, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "output.weight", "offset": 7599718400, "shape": [ 5120, 32064 ], "size": 24576000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7734387200, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7734407680, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.35.ffn_up.weight", "offset": 7791751680, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.35.ffn_norm.weight", "offset": 7906439680, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7906460160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7906480640, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.36.ffn_up.weight", "offset": 7963824640, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 8078512640, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 8078533120, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.36.attn_qkv.weight", "offset": 8094917120, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.37.attn_norm.weight", "offset": 8119493120, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8119513600, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.37.ffn_up.weight", "offset": 8176857600, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.37.ffn_norm.weight", "offset": 8291545600, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 8291566080, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.37.attn_qkv.weight", "offset": 8307950080, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.38.attn_norm.weight", "offset": 8332526080, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8332546560, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.38.ffn_up.weight", "offset": 8389890560, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.38.ffn_norm.weight", "offset": 8504578560, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 8504599040, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.38.attn_qkv.weight", "offset": 8520983040, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 8545559040, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8545579520, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.39.ffn_up.weight", "offset": 8602923520, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.39.ffn_norm.weight", "offset": 8717611520, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 8717632000, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.39.attn_qkv.weight", "offset": 8734016000, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "output_norm.weight", "offset": 8758592000, "shape": [ 5120 ], "size": 24576000, "type": 0 } ], "version": 3 }