phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 1, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 32, "phi3.attention.head_count_kv": 32, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 32, "phi3.context_length": 4096, "phi3.embedding_length": 3072, "phi3.feed_forward_length": 8192, "phi3.rope.dimension_count": 96, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 3821079552, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 197001216, "shape": [ 3072 ], "size": 197001216, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 197013504, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 247345152, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 348008448, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 348020736, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.0.attn_qkv.weight", "offset": 366895104, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 423518208, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 423530496, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 473862144, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 574525440, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 574537728, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.1.attn_qkv.weight", "offset": 593412096, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 650035200, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 650047488, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 700379136, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 801042432, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 801054720, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.10.attn_qkv.weight", "offset": 819929088, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 876552192, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 876564480, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 926896128, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 1027559424, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1027571712, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.11.attn_qkv.weight", "offset": 1046446080, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 1103069184, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1103081472, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 1153413120, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 1254076416, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1254088704, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.12.attn_qkv.weight", "offset": 1272963072, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 1329586176, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1329598464, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 1379930112, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 1480593408, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1480605696, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.13.attn_qkv.weight", "offset": 1499480064, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 1556103168, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1556115456, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 1606447104, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 1707110400, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1707122688, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.14.attn_qkv.weight", "offset": 1725997056, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 1782620160, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1782632448, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 1832964096, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 1933627392, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 1933639680, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.15.attn_qkv.weight", "offset": 1952514048, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 2009137152, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2009149440, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 2059481088, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 2160144384, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2160156672, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.16.attn_qkv.weight", "offset": 2179031040, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 2235654144, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2235666432, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 2285998080, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 2386661376, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2386673664, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.17.attn_qkv.weight", "offset": 2405548032, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 2462171136, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2462183424, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 2512515072, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 2613178368, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2613190656, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.18.attn_qkv.weight", "offset": 2632065024, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 2688688128, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2688700416, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 2739032064, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 2839695360, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2839707648, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.19.attn_qkv.weight", "offset": 2858582016, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 2915205120, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2915217408, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 2965549056, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 3066212352, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 3066224640, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.2.attn_qkv.weight", "offset": 3085099008, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 3141722112, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3141734400, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 3192066048, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 3292729344, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3292741632, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.20.attn_qkv.weight", "offset": 3311616000, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 3368239104, "shape": [ 3072, 3072 ], "size": 56623104, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 3387113472, "shape": [ 3072 ], "size": 18874368, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 3387125760, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 3437457408, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 3538120704, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 3538132992, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.3.attn_qkv.weight", "offset": 3557007360, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3613630464, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3613642752, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3663974400, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 3764637696, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 3764649984, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.4.attn_qkv.weight", "offset": 3783524352, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 3840147456, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3840159744, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 3890491392, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 3991154688, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 3991166976, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.5.attn_qkv.weight", "offset": 4010041344, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 4066664448, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4066676736, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 4117008384, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 4217671680, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 4217683968, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.6.attn_qkv.weight", "offset": 4236558336, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 4293181440, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4293193728, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 4343525376, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 4444188672, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 4444200960, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.7.attn_qkv.weight", "offset": 4463075328, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 4519698432, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4519710720, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 4570042368, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 4670705664, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 4670717952, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.8.attn_qkv.weight", "offset": 4689592320, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 4746215424, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 4746227712, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 4796559360, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 4897222656, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 4897234944, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.9.attn_qkv.weight", "offset": 4916109312, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "output.weight", "offset": 4972732416, "shape": [ 3072, 32064 ], "size": 56623104, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 5169733632, "shape": [ 3072 ], "size": 197001216, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5169745920, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 5220077568, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 5320740864, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 5320753152, "shape": [ 3072, 9216 ], "size": 12288, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 5377376256, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5377388544, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 5427720192, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 5528383488, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5528395776, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.22.attn_qkv.weight", "offset": 5547270144, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 5603893248, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5603905536, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 5654237184, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 5754900480, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5754912768, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.23.attn_qkv.weight", "offset": 5773787136, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 5830410240, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5830422528, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 5880754176, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 5981417472, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5981429760, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.24.attn_qkv.weight", "offset": 6000304128, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 6056927232, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6056939520, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 6107271168, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 6207934464, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6207946752, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.25.attn_qkv.weight", "offset": 6226821120, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 6283444224, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6283456512, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 6333788160, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 6434451456, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6434463744, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.26.attn_qkv.weight", "offset": 6453338112, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 6509961216, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6509973504, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 6560305152, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 6660968448, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6660980736, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.27.attn_qkv.weight", "offset": 6679855104, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 6736478208, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6736490496, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 6786822144, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 6887485440, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.28.attn_output.weight", "offset": 6887497728, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.28.attn_qkv.weight", "offset": 6906372096, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 6962995200, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6963007488, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 7013339136, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 7114002432, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7114014720, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.29.attn_qkv.weight", "offset": 7132889088, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 7189512192, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7189524480, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 7239856128, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 7340519424, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7340531712, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.30.attn_qkv.weight", "offset": 7359406080, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 7416029184, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7416041472, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 7466373120, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 7567036416, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7567048704, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.31.attn_qkv.weight", "offset": 7585923072, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "output_norm.weight", "offset": 7642546176, "shape": [ 3072 ], "size": 56623104, "type": 0 } ], "version": 3 }