Gemma is a family of lightweight, state-of-the-art open models built by Google DeepMind. Updated to version 1.1

2B 7B

3.5M Pulls Updated 2 months ago

102 Tags

ef311de6af9d · 5.0GB
{ "metadata": { "gemma.attention.head_count": 16, "gemma.attention.head_count_kv": 16, "gemma.attention.key_length": 256, "gemma.attention.layer_norm_rms_epsilon": 0.000001, "gemma.attention.value_length": 256, "gemma.block_count": 28, "gemma.context_length": 8192, "gemma.embedding_length": 3072, "gemma.feed_forward_length": 24576, "general.architecture": "gemma", "general.file_type": 2, "general.name": "gemma-1.1-7b-it", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 8537680896, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 645120000, "shape": [ 3072 ], "size": 645120000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 645132288, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 687599616, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 730066944, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 772534272, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 772546560, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 779624448, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 786702336, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 793780224, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 800858112, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 800870400, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 843337728, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 885805056, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 928272384, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 928284672, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 935362560, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 942440448, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 949518336, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 956596224, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 956608512, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 999075840, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 1041543168, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 1084010496, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1084022784, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 1091100672, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 1098178560, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 1105256448, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 1112334336, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1112346624, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 1154813952, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 1197281280, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 1239748608, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1239760896, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 1246838784, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 1253916672, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 1260994560, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 1268072448, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1268084736, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 1310552064, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 1353019392, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1395486720, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1395499008, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 1402576896, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 1409654784, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 1416732672, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1423810560, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1423822848, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 1466290176, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1508757504, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1551224832, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1551237120, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 1558315008, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 1565392896, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 1572470784, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.6.attn_k.weight", "offset": 1579548672, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1586626560, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 1593704448, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 1600782336, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1607860224, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1607872512, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 1650339840, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1692807168, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1735274496, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1735286784, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 1742364672, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 1749442560, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 1756520448, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1763598336, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1763610624, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1806077952, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1848545280, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1891012608, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1891024896, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 1898102784, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 1905180672, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 1912258560, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1919336448, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1919348736, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 1961816064, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 2004283392, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 2046750720, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 2046763008, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 2053840896, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 2060918784, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 2067996672, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 2075074560, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2075086848, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 2117554176, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 2160021504, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 2202488832, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2202501120, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 2209579008, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 2216656896, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 2223734784, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 2230812672, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2230824960, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 2273292288, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 2315759616, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 2358226944, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2358239232, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 2365317120, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 2372395008, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 2379472896, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.15.attn_k.weight", "offset": 2386550784, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 2393628672, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 2400706560, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 2407784448, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 2414862336, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2414874624, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 2457341952, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 2499809280, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 2542276608, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2542288896, "shape": [ 3072 ], "size": 12288, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2542301184, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 2584768512, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 2627235840, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 2669703168, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2669715456, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 2676793344, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 2683871232, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 2690949120, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 2698027008, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2698039296, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 2740506624, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 2782973952, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 2825441280, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2825453568, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 2832531456, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 2839609344, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 2846687232, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2853765120, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2853777408, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 2896244736, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2938712064, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2981179392, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2981191680, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 2988269568, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 2995347456, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 3002425344, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 3009503232, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3009515520, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 3051982848, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 3094450176, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 3136917504, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 3136929792, "shape": [ 3072 ], "size": 12288, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3136942080, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 3179409408, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 3221876736, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 3264344064, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3264356352, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 3271434240, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 3278512128, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 3285590016, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 3292667904, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3292680192, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 3335147520, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 3377614848, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 3420082176, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3420094464, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 3427172352, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 3434250240, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 3441328128, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 3448406016, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3448418304, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 3490885632, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 3533352960, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 3575820288, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3575832576, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 3582910464, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.18.attn_q.weight", "offset": 3589988352, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 3597066240, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 3604144128, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3604156416, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 3646623744, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 3689091072, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 3731558400, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3731570688, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 3738648576, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 3745726464, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 3752804352, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 3759882240, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3759894528, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 3802361856, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 3844829184, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 3887296512, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3887308800, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 3894386688, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 3901464576, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 3908542464, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 3915620352, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3915632640, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 3958099968, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 4000567296, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 4043034624, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4043046912, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 4050124800, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 4057202688, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 4064280576, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 4071358464, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4071370752, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 4113838080, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 4156305408, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 4198772736, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4198785024, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 4205862912, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 4212940800, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 4220018688, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 4227096576, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4227108864, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 4269576192, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 4312043520, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 4354510848, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4354523136, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 4361601024, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 4368678912, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 4375756800, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.24.attn_k.weight", "offset": 4382834688, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 4389912576, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 4396990464, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 4404068352, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 4411146240, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4411158528, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 4453625856, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 4496093184, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 4538560512, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.25.attn_norm.weight", "offset": 4538572800, "shape": [ 3072 ], "size": 12288, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4538585088, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 4581052416, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 4623519744, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 4665987072, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4665999360, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 4673077248, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 4680155136, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 4687233024, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 4694310912, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4694323200, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 4736790528, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 4779257856, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 4821725184, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4821737472, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 4828815360, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 4835893248, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 4842971136, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 4850049024, "shape": [ 3072 ], "size": 7077888, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4850061312, "shape": [ 24576, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 4892528640, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 4934995968, "shape": [ 3072, 24576 ], "size": 42467328, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 4977463296, "shape": [ 3072 ], "size": 42467328, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4977475584, "shape": [ 3072, 4096 ], "size": 12288, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 4984553472, "shape": [ 4096, 3072 ], "size": 7077888, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 4991631360, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 4998709248, "shape": [ 3072, 4096 ], "size": 7077888, "type": 2 }, { "name": "output_norm.weight", "offset": 5005787136, "shape": [ 3072 ], "size": 7077888, "type": 0 } ], "version": 3 }