codegemma

{ "metadata": { "gemma.attention.head_count": 8, "gemma.attention.head_count_kv": 1, "gemma.attention.key_length": 256, "gemma.attention.layer_norm_rms_epsilon": 0.000001, "gemma.attention.value_length": 256, "gemma.block_count": 18, "gemma.context_length": 8192, "gemma.embedding_length": 2048, "gemma.feed_forward_length": 16384, "general.architecture": "gemma", "general.file_type": 2, "general.name": "codegemma-1.1-2b", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)" }, "num_params": 2506172416, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 430080000, "shape": [ 2048 ], "size": 430080000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 430088192, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 448962560, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 467836928, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 486711296, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 486719488, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 487014400, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 489373696, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 491732992, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 492027904, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 492036096, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 510910464, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 529784832, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 548659200, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 548667392, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 548962304, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 551321600, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 553680896, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 553975808, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 553984000, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 572858368, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 591732736, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 610607104, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 610615296, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 610910208, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 613269504, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 615628800, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 615923712, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 615931904, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 634806272, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 653680640, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 672555008, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 672563200, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 672858112, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.11.attn_q.weight", "offset": 675217408, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 677576704, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 677871616, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 677879808, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 696754176, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 715628544, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 734502912, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 734511104, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 734806016, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 737165312, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 739524608, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 739819520, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 739827712, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 758702080, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 777576448, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 796450816, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 796459008, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 796753920, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 799113216, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 801472512, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 801767424, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 801775616, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 820649984, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 839524352, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 858398720, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 858406912, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 858701824, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 861061120, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 863420416, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 863715328, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 863723520, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 882597888, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 901472256, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 920346624, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 920354816, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 920649728, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 923009024, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 925368320, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 925663232, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 925671424, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 944545792, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 963420160, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 982294528, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 982302720, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 982597632, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.16.attn_q.weight", "offset": 984956928, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 987316224, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 987611136, "shape": [ 2048, 16384 ], "size": 294912, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 1006485504, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.17.attn_k.weight", "offset": 1025359872, "shape": [ 2048, 256 ], "size": 18874368, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 1025654784, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 1028014080, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 1030373376, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 1030668288, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1030676480, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 1049550848, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 1068425216, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 1087299584, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1087307776, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 1087602688, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 1089961984, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 1092321280, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 1092616192, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1092624384, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 1111498752, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 1130373120, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 1149247488, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1149255680, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 1149550592, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 1151909888, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.3.attn_v.weight", "offset": 1154269184, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 1154564096, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1154572288, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 1173446656, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 1192321024, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1211195392, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1211203584, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 1211498496, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 1213857792, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 1216217088, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1216512000, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1216520192, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 1235394560, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1254268928, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1273143296, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1273151488, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 1273446400, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 1275805696, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 1278164992, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 1278459904, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1278468096, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 1297342464, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1316216832, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1335091200, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1335099392, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1335394304, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 1337753600, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 1340112896, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 1340407808, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1340416000, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 1359290368, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1378164736, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1397039104, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1397047296, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 1397342208, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 1399701504, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 1402060800, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1402355712, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1402363904, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 1421238272, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 1440112640, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 1458987008, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1458995200, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 1459290112, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 1461649408, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 1464008704, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 1464303616, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1464311808, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 1483186176, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 1502060544, "shape": [ 2048, 16384 ], "size": 18874368, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 1520934912, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1520943104, "shape": [ 2048, 256 ], "size": 8192, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 1521238016, "shape": [ 2048, 2048 ], "size": 294912, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 1523597312, "shape": [ 2048, 2048 ], "size": 2359296, "type": 2 }, { "name": "blk.9.attn_v.weight", "offset": 1525956608, "shape": [ 2048, 256 ], "size": 2359296, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 1526251520, "shape": [ 2048 ], "size": 294912, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1526259712, "shape": [ 16384, 2048 ], "size": 8192, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 1545134080, "shape": [ 2048 ], "size": 18874368, "type": 0 }, { "name": "output_norm.weight", "offset": 1545142272, "shape": [ 2048 ], "size": 8192, "type": 0 } ], "version": 3 }