latest
1.6GB
An expert in DevSecOps. This AI is able to provision well-architected and secure CSP infrastrucure.
3B
47 Pulls Updated 3 months ago
4eca7304a07a · 1.6GB
{
"metadata": {
"general.architecture": "phi2",
"general.file_type": 2,
"general.name": "Phi2",
"general.quantization_version": 2,
"phi2.attention.head_count": 32,
"phi2.attention.head_count_kv": 32,
"phi2.attention.layer_norm_epsilon": 0.00001,
"phi2.block_count": 32,
"phi2.context_length": 2048,
"phi2.embedding_length": 2560,
"phi2.feed_forward_length": 10240,
"phi2.rope.dimension_count": 32,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.bos_token_id": 50256,
"tokenizer.ggml.eos_token_id": 50295,
"tokenizer.ggml.merges": "... (50000 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 50256,
"tokenizer.ggml.token_type": "... (51200 values)",
"tokenizer.ggml.tokens": "... (51200 values)",
"tokenizer.ggml.unknown_token_id": 50256
},
"num_params": 2779683840,
"tensors": [
{
"name": "blk.0.attn_norm.bias",
"offset": 73728000,
"shape": [
2560
],
"size": 73728000,
"type": 0
},
{
"name": "blk.0.attn_norm.weight",
"offset": 73738240,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.0.attn_qkv.bias",
"offset": 73748480,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.0.attn_qkv.weight",
"offset": 73779200,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.0.attn_output.bias",
"offset": 84838400,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.0.attn_output.weight",
"offset": 84848640,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.0.ffn_up.bias",
"offset": 88535040,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.0.ffn_up.weight",
"offset": 88576000,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.0.ffn_down.bias",
"offset": 103321600,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 103331840,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.1.attn_norm.bias",
"offset": 118077440,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.1.attn_norm.weight",
"offset": 118087680,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.1.attn_qkv.bias",
"offset": 118097920,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.1.attn_qkv.weight",
"offset": 118128640,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.1.attn_output.bias",
"offset": 129187840,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.1.attn_output.weight",
"offset": 129198080,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.1.ffn_up.bias",
"offset": 132884480,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.1.ffn_up.weight",
"offset": 132925440,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.1.ffn_down.bias",
"offset": 147671040,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 147681280,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.10.attn_norm.bias",
"offset": 162426880,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.10.attn_norm.weight",
"offset": 162437120,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.10.attn_qkv.bias",
"offset": 162447360,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.10.attn_qkv.weight",
"offset": 162478080,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.10.attn_output.bias",
"offset": 173537280,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.10.attn_output.weight",
"offset": 173547520,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.10.ffn_up.bias",
"offset": 177233920,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.10.ffn_up.weight",
"offset": 177274880,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.10.ffn_down.bias",
"offset": 192020480,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 192030720,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.11.attn_norm.bias",
"offset": 206776320,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.11.attn_norm.weight",
"offset": 206786560,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.11.attn_qkv.bias",
"offset": 206796800,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.11.attn_qkv.weight",
"offset": 206827520,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.11.attn_output.bias",
"offset": 217886720,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.11.attn_output.weight",
"offset": 217896960,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.11.ffn_up.bias",
"offset": 221583360,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.11.ffn_up.weight",
"offset": 221624320,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.11.ffn_down.bias",
"offset": 236369920,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 236380160,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.12.attn_norm.bias",
"offset": 251125760,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.12.attn_norm.weight",
"offset": 251136000,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.12.attn_qkv.bias",
"offset": 251146240,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.12.attn_qkv.weight",
"offset": 251176960,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.12.attn_output.bias",
"offset": 262236160,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.12.attn_output.weight",
"offset": 262246400,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.12.ffn_up.bias",
"offset": 265932800,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.12.ffn_up.weight",
"offset": 265973760,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.12.ffn_down.bias",
"offset": 280719360,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 280729600,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.13.attn_norm.bias",
"offset": 295475200,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 295485440,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.13.attn_qkv.bias",
"offset": 295495680,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.13.attn_qkv.weight",
"offset": 295526400,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.13.attn_output.bias",
"offset": 306585600,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.13.attn_output.weight",
"offset": 306595840,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.13.ffn_up.bias",
"offset": 310282240,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.13.ffn_up.weight",
"offset": 310323200,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.13.ffn_down.bias",
"offset": 325068800,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 325079040,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.14.attn_norm.bias",
"offset": 339824640,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.14.attn_norm.weight",
"offset": 339834880,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.14.attn_qkv.bias",
"offset": 339845120,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.14.attn_qkv.weight",
"offset": 339875840,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.14.attn_output.bias",
"offset": 350935040,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.14.attn_output.weight",
"offset": 350945280,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.14.ffn_up.bias",
"offset": 354631680,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.14.ffn_up.weight",
"offset": 354672640,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.14.ffn_down.bias",
"offset": 369418240,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 369428480,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.15.attn_norm.bias",
"offset": 384174080,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.15.attn_norm.weight",
"offset": 384184320,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.15.attn_qkv.bias",
"offset": 384194560,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.15.attn_qkv.weight",
"offset": 384225280,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.15.attn_output.bias",
"offset": 395284480,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.15.attn_output.weight",
"offset": 395294720,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.15.ffn_up.bias",
"offset": 398981120,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.15.ffn_up.weight",
"offset": 399022080,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.15.ffn_down.bias",
"offset": 413767680,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 413777920,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.16.attn_norm.bias",
"offset": 428523520,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.16.attn_norm.weight",
"offset": 428533760,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.16.attn_qkv.bias",
"offset": 428544000,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.16.attn_qkv.weight",
"offset": 428574720,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.16.attn_output.bias",
"offset": 439633920,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.16.attn_output.weight",
"offset": 439644160,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.16.ffn_up.bias",
"offset": 443330560,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.16.ffn_up.weight",
"offset": 443371520,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.16.ffn_down.bias",
"offset": 458117120,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 458127360,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.17.attn_norm.bias",
"offset": 472872960,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 472883200,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.17.attn_qkv.bias",
"offset": 472893440,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.17.attn_qkv.weight",
"offset": 472924160,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.17.attn_output.bias",
"offset": 483983360,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.17.attn_output.weight",
"offset": 483993600,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.17.ffn_up.bias",
"offset": 487680000,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.17.ffn_up.weight",
"offset": 487720960,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.17.ffn_down.bias",
"offset": 502466560,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 502476800,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.18.attn_norm.bias",
"offset": 517222400,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.18.attn_norm.weight",
"offset": 517232640,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.18.attn_qkv.bias",
"offset": 517242880,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.18.attn_qkv.weight",
"offset": 517273600,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.18.attn_output.bias",
"offset": 528332800,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.18.attn_output.weight",
"offset": 528343040,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.18.ffn_up.bias",
"offset": 532029440,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.18.ffn_up.weight",
"offset": 532070400,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.18.ffn_down.bias",
"offset": 546816000,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 546826240,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.19.attn_norm.bias",
"offset": 561571840,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.19.attn_norm.weight",
"offset": 561582080,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.19.attn_qkv.bias",
"offset": 561592320,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.19.attn_qkv.weight",
"offset": 561623040,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.19.attn_output.bias",
"offset": 572682240,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.19.attn_output.weight",
"offset": 572692480,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.19.ffn_up.bias",
"offset": 576378880,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.19.ffn_up.weight",
"offset": 576419840,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.19.ffn_down.bias",
"offset": 591165440,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 591175680,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.2.attn_norm.bias",
"offset": 605921280,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.2.attn_norm.weight",
"offset": 605931520,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.2.attn_qkv.bias",
"offset": 605941760,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.2.attn_qkv.weight",
"offset": 605972480,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.2.attn_output.bias",
"offset": 617031680,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.2.attn_output.weight",
"offset": 617041920,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.2.ffn_up.bias",
"offset": 620728320,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.2.ffn_up.weight",
"offset": 620769280,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.2.ffn_down.bias",
"offset": 635514880,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 635525120,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.20.attn_norm.bias",
"offset": 650270720,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.20.attn_norm.weight",
"offset": 650280960,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.20.attn_qkv.bias",
"offset": 650291200,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.20.attn_qkv.weight",
"offset": 650321920,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.20.attn_output.bias",
"offset": 661381120,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.20.attn_output.weight",
"offset": 661391360,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.20.ffn_up.bias",
"offset": 665077760,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.20.ffn_up.weight",
"offset": 665118720,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.20.ffn_down.bias",
"offset": 679864320,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 679874560,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.21.attn_norm.bias",
"offset": 694620160,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 694630400,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.21.attn_qkv.bias",
"offset": 694640640,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.21.attn_qkv.weight",
"offset": 694671360,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.21.attn_output.bias",
"offset": 705730560,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.21.attn_output.weight",
"offset": 705740800,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.21.ffn_up.bias",
"offset": 709427200,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.21.ffn_up.weight",
"offset": 709468160,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.21.ffn_down.bias",
"offset": 724213760,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 724224000,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.22.attn_norm.bias",
"offset": 738969600,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.22.attn_norm.weight",
"offset": 738979840,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.22.attn_qkv.bias",
"offset": 738990080,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.22.attn_qkv.weight",
"offset": 739020800,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.22.attn_output.bias",
"offset": 750080000,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.22.attn_output.weight",
"offset": 750090240,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.22.ffn_up.bias",
"offset": 753776640,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.22.ffn_up.weight",
"offset": 753817600,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.22.ffn_down.bias",
"offset": 768563200,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 768573440,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.23.attn_norm.bias",
"offset": 783319040,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.23.attn_norm.weight",
"offset": 783329280,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.23.attn_qkv.bias",
"offset": 783339520,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.23.attn_qkv.weight",
"offset": 783370240,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.23.attn_output.bias",
"offset": 794429440,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.23.attn_output.weight",
"offset": 794439680,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.23.ffn_up.bias",
"offset": 798126080,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.23.ffn_up.weight",
"offset": 798167040,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.23.ffn_down.bias",
"offset": 812912640,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 812922880,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.24.attn_norm.bias",
"offset": 827668480,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.24.attn_norm.weight",
"offset": 827678720,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.24.attn_qkv.bias",
"offset": 827688960,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.24.attn_qkv.weight",
"offset": 827719680,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.24.attn_output.bias",
"offset": 838778880,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.24.attn_output.weight",
"offset": 838789120,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.24.ffn_up.bias",
"offset": 842475520,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.24.ffn_up.weight",
"offset": 842516480,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.24.ffn_down.bias",
"offset": 857262080,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 857272320,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.25.attn_norm.bias",
"offset": 872017920,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.25.attn_norm.weight",
"offset": 872028160,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.25.attn_qkv.bias",
"offset": 872038400,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.25.attn_qkv.weight",
"offset": 872069120,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.25.attn_output.bias",
"offset": 883128320,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.25.attn_output.weight",
"offset": 883138560,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.25.ffn_up.bias",
"offset": 886824960,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.25.ffn_up.weight",
"offset": 886865920,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.25.ffn_down.bias",
"offset": 901611520,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 901621760,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.26.attn_norm.bias",
"offset": 916367360,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 916377600,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.26.attn_qkv.bias",
"offset": 916387840,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.26.attn_qkv.weight",
"offset": 916418560,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.26.attn_output.bias",
"offset": 927477760,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.26.attn_output.weight",
"offset": 927488000,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.26.ffn_up.bias",
"offset": 931174400,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.26.ffn_up.weight",
"offset": 931215360,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.26.ffn_down.bias",
"offset": 945960960,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 945971200,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.27.attn_norm.bias",
"offset": 960716800,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.27.attn_norm.weight",
"offset": 960727040,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.27.attn_qkv.bias",
"offset": 960737280,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.27.attn_qkv.weight",
"offset": 960768000,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.27.attn_output.bias",
"offset": 971827200,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 971837440,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.27.ffn_up.bias",
"offset": 975523840,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.27.ffn_up.weight",
"offset": 975564800,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.27.ffn_down.bias",
"offset": 990310400,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 990320640,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.28.attn_norm.bias",
"offset": 1005066240,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.28.attn_norm.weight",
"offset": 1005076480,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.28.attn_qkv.bias",
"offset": 1005086720,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.28.attn_qkv.weight",
"offset": 1005117440,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.28.attn_output.bias",
"offset": 1016176640,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.28.attn_output.weight",
"offset": 1016186880,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.28.ffn_up.bias",
"offset": 1019873280,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.28.ffn_up.weight",
"offset": 1019914240,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.28.ffn_down.bias",
"offset": 1034659840,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 1034670080,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.29.attn_norm.bias",
"offset": 1049415680,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 1049425920,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.29.attn_qkv.bias",
"offset": 1049436160,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.29.attn_qkv.weight",
"offset": 1049466880,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.29.attn_output.bias",
"offset": 1060526080,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.29.attn_output.weight",
"offset": 1060536320,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.29.ffn_up.bias",
"offset": 1064222720,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.29.ffn_up.weight",
"offset": 1064263680,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.29.ffn_down.bias",
"offset": 1079009280,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 1079019520,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.3.attn_norm.bias",
"offset": 1093765120,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1093775360,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.3.attn_qkv.bias",
"offset": 1093785600,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.3.attn_qkv.weight",
"offset": 1093816320,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.3.attn_output.bias",
"offset": 1104875520,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.3.attn_output.weight",
"offset": 1104885760,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.3.ffn_up.bias",
"offset": 1108572160,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1108613120,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.3.ffn_down.bias",
"offset": 1123358720,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1123368960,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.30.attn_norm.bias",
"offset": 1138114560,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 1138124800,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.4.attn_norm.bias",
"offset": 1138135040,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1138145280,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.4.attn_qkv.bias",
"offset": 1138155520,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.4.attn_qkv.weight",
"offset": 1138186240,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.4.attn_output.bias",
"offset": 1149245440,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.4.attn_output.weight",
"offset": 1149255680,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.4.ffn_up.bias",
"offset": 1152942080,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1152983040,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.4.ffn_down.bias",
"offset": 1167728640,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1167738880,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.5.attn_norm.bias",
"offset": 1182484480,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1182494720,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.5.attn_qkv.bias",
"offset": 1182504960,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.5.attn_qkv.weight",
"offset": 1182535680,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.5.attn_output.bias",
"offset": 1193594880,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.5.attn_output.weight",
"offset": 1193605120,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.5.ffn_up.bias",
"offset": 1197291520,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1197332480,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.5.ffn_down.bias",
"offset": 1212078080,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1212088320,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.6.attn_norm.bias",
"offset": 1226833920,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1226844160,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.6.attn_qkv.bias",
"offset": 1226854400,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.6.attn_qkv.weight",
"offset": 1226885120,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.6.attn_output.bias",
"offset": 1237944320,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.6.attn_output.weight",
"offset": 1237954560,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.6.ffn_up.bias",
"offset": 1241640960,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1241681920,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.6.ffn_down.bias",
"offset": 1256427520,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1256437760,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.7.attn_norm.bias",
"offset": 1271183360,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1271193600,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.7.attn_qkv.bias",
"offset": 1271203840,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.7.attn_qkv.weight",
"offset": 1271234560,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.7.attn_output.bias",
"offset": 1282293760,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.7.attn_output.weight",
"offset": 1282304000,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.7.ffn_up.bias",
"offset": 1285990400,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1286031360,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.7.ffn_down.bias",
"offset": 1300776960,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1300787200,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.8.attn_norm.bias",
"offset": 1315532800,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1315543040,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.8.attn_qkv.bias",
"offset": 1315553280,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.8.attn_qkv.weight",
"offset": 1315584000,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.8.attn_output.bias",
"offset": 1326643200,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.8.attn_output.weight",
"offset": 1326653440,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.8.ffn_up.bias",
"offset": 1330339840,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1330380800,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.8.ffn_down.bias",
"offset": 1345126400,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1345136640,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.9.attn_norm.bias",
"offset": 1359882240,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.9.attn_norm.weight",
"offset": 1359892480,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.9.attn_qkv.bias",
"offset": 1359902720,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.9.attn_qkv.weight",
"offset": 1359933440,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.9.attn_output.bias",
"offset": 1370992640,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.9.attn_output.weight",
"offset": 1371002880,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.9.ffn_up.bias",
"offset": 1374689280,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.9.ffn_up.weight",
"offset": 1374730240,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.9.ffn_down.bias",
"offset": 1389475840,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 1389486080,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "output.bias",
"offset": 1404231680,
"shape": [
51200
],
"size": 14745600,
"type": 0
},
{
"name": "output.weight",
"offset": 1404436480,
"shape": [
2560,
51200
],
"size": 204800,
"type": 14
},
{
"name": "output_norm.bias",
"offset": 1511956480,
"shape": [
2560
],
"size": 107520000,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 1511966720,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.30.attn_qkv.bias",
"offset": 1511976960,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.30.attn_qkv.weight",
"offset": 1512007680,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.30.attn_output.bias",
"offset": 1523066880,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.30.attn_output.weight",
"offset": 1523077120,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.30.ffn_up.bias",
"offset": 1526763520,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.30.ffn_up.weight",
"offset": 1526804480,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.30.ffn_down.bias",
"offset": 1541550080,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 1541560320,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.31.attn_norm.bias",
"offset": 1556305920,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.31.attn_norm.weight",
"offset": 1556316160,
"shape": [
2560
],
"size": 10240,
"type": 0
},
{
"name": "blk.31.attn_qkv.bias",
"offset": 1556326400,
"shape": [
7680
],
"size": 10240,
"type": 0
},
{
"name": "blk.31.attn_qkv.weight",
"offset": 1556357120,
"shape": [
2560,
7680
],
"size": 30720,
"type": 2
},
{
"name": "blk.31.attn_output.bias",
"offset": 1567416320,
"shape": [
2560
],
"size": 11059200,
"type": 0
},
{
"name": "blk.31.attn_output.weight",
"offset": 1567426560,
"shape": [
2560,
2560
],
"size": 10240,
"type": 2
},
{
"name": "blk.31.ffn_up.bias",
"offset": 1571112960,
"shape": [
10240
],
"size": 3686400,
"type": 0
},
{
"name": "blk.31.ffn_up.weight",
"offset": 1571153920,
"shape": [
2560,
10240
],
"size": 40960,
"type": 2
},
{
"name": "blk.31.ffn_down.bias",
"offset": 1585899520,
"shape": [
2560
],
"size": 14745600,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 1585909760,
"shape": [
10240,
2560
],
"size": 10240,
"type": 2
}
],
"version": 3
}