qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 3, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 200047104, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 208648704, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 217250304, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 217256448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 217257472, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 217503232, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.0.attn_q.bias", "offset": 218977792, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 218983936, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.0.attn_v.bias", "offset": 220458496, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 220459520, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 220705280, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 220711424, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 229313024, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 237914624, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 246516224, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 246522368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 246523392, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 246769152, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.1.attn_q.bias", "offset": 248243712, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 248249856, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.1.attn_v.bias", "offset": 249724416, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 249725440, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 249971200, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 249977344, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 258578944, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 267180544, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 275782144, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 275788288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 275789312, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 276035072, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.10.attn_q.bias", "offset": 277509632, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 277515776, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.10.attn_v.bias", "offset": 278990336, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 278991360, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 279237120, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 279243264, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 287844864, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 296446464, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 305048064, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 305054208, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 305055232, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 305300992, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.11.attn_q.bias", "offset": 306775552, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 306781696, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.11.attn_v.bias", "offset": 308256256, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 308257280, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 308503040, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 308509184, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 317110784, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 325712384, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 334313984, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 334320128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 334321152, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 334566912, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.12.attn_q.bias", "offset": 336041472, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 336047616, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.12.attn_v.bias", "offset": 337522176, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 337523200, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 337768960, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 337775104, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 346376704, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 354978304, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 363579904, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 363586048, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 363587072, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 363832832, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.13.attn_q.bias", "offset": 365307392, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 365313536, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.13.attn_v.bias", "offset": 366788096, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 366789120, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 367034880, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 367041024, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 375642624, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 384244224, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 392845824, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 392851968, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 392852992, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 393098752, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.14.attn_q.bias", "offset": 394573312, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 394579456, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.14.attn_v.bias", "offset": 396054016, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 396055040, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 396300800, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 396306944, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 404908544, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 413510144, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 422111744, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 422117888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 422118912, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 422364672, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.15.attn_q.bias", "offset": 423839232, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 423845376, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.15.attn_v.bias", "offset": 425319936, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 425320960, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 425566720, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 425572864, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 434174464, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 442776064, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 451377664, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 451383808, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 451384832, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 451630592, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.16.attn_q.bias", "offset": 453105152, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 453111296, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.16.attn_v.bias", "offset": 454585856, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 454586880, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 454832640, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 454838784, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 463440384, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 472041984, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 480643584, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 480649728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 480650752, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 480896512, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.17.attn_q.bias", "offset": 482371072, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 482377216, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.17.attn_v.bias", "offset": 483851776, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 483852800, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 484098560, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 484104704, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 492706304, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 501307904, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 509909504, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 509915648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 509916672, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 510162432, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.18.attn_q.bias", "offset": 511636992, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 511643136, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.18.attn_v.bias", "offset": 513117696, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 513118720, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 513364480, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 513370624, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 521972224, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 530573824, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 539175424, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 539181568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 539182592, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 539428352, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.19.attn_q.bias", "offset": 540902912, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 540909056, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.19.attn_v.bias", "offset": 542383616, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 542384640, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 542630400, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 542636544, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 551238144, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 559839744, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 568441344, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 568447488, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 568448512, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 568694272, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.2.attn_q.bias", "offset": 570168832, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 570174976, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.2.attn_v.bias", "offset": 571649536, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 571650560, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 571896320, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 571902464, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 580504064, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 589105664, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 597707264, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 597713408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 597714432, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 597960192, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.20.attn_q.bias", "offset": 599434752, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 599440896, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.20.attn_v.bias", "offset": 600915456, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 600916480, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 601162240, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 601168384, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 609769984, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 618371584, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 626973184, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 626979328, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 626980352, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 627226112, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.21.attn_q.bias", "offset": 628700672, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 628706816, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.21.attn_v.bias", "offset": 630181376, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 630182400, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 630428160, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 630434304, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 639035904, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 647637504, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 656239104, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 656245248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 656246272, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 656492032, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.22.attn_q.bias", "offset": 657966592, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 657972736, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.22.attn_v.bias", "offset": 659447296, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 659448320, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 659694080, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 659700224, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 668301824, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 676903424, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 685505024, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 685511168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 685512192, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 685757952, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.23.attn_q.bias", "offset": 687232512, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 687238656, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.23.attn_v.bias", "offset": 688713216, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 688714240, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 688960000, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 688966144, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 697567744, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 706169344, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 714770944, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 714777088, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 714778112, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 715023872, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.24.attn_q.bias", "offset": 716498432, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 716504576, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.24.attn_v.bias", "offset": 717979136, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 717980160, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 718225920, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 718232064, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 726833664, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 735435264, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 744036864, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 744043008, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 744044032, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 744289792, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.25.attn_q.bias", "offset": 745764352, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 745770496, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.25.attn_v.bias", "offset": 747245056, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 747246080, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 747491840, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 747497984, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 756099584, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 764701184, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 773302784, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 773308928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 773309952, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 773555712, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.26.attn_q.bias", "offset": 775030272, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 775036416, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.26.attn_v.bias", "offset": 776510976, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 776512000, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 776757760, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 776763904, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 785365504, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 793967104, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 802568704, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 802574848, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 802575872, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 802821632, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.27.attn_q.bias", "offset": 804296192, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 804302336, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.27.attn_v.bias", "offset": 805776896, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 805777920, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 806023680, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 806029824, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 814631424, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 823233024, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 831834624, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 831840768, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 831841792, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 832087552, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.3.attn_q.bias", "offset": 833562112, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 833568256, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.3.attn_v.bias", "offset": 835042816, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 835043840, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 835289600, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 835295744, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 843897344, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 852498944, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 861100544, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 861106688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 861107712, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 861353472, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.4.attn_q.bias", "offset": 862828032, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 862834176, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.4.attn_v.bias", "offset": 864308736, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 864309760, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 864555520, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 864561664, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 873163264, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 881764864, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 890366464, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 890372608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 890373632, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 890619392, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.5.attn_q.bias", "offset": 892093952, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 892100096, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.5.attn_v.bias", "offset": 893574656, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 893575680, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 893821440, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 893827584, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 902429184, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 911030784, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 919632384, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 919638528, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 919639552, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 919885312, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.6.attn_q.bias", "offset": 921359872, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 921366016, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.6.attn_v.bias", "offset": 922840576, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 922841600, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 923087360, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 923093504, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 931695104, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 940296704, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 948898304, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 948904448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 948905472, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 949151232, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.7.attn_q.bias", "offset": 950625792, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 950631936, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.7.attn_v.bias", "offset": 952106496, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 952107520, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 952353280, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 952359424, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 960961024, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 969562624, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 978164224, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 978170368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 978171392, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 978417152, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.8.attn_q.bias", "offset": 979891712, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 979897856, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.8.attn_v.bias", "offset": 981372416, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 981373440, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 981619200, "shape": [ 1536 ], "size": 245760, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 981625344, "shape": [ 8960, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 990226944, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 998828544, "shape": [ 1536, 8960 ], "size": 8601600, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 1007430144, "shape": [ 1536 ], "size": 8601600, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1007436288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1007437312, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 1007683072, "shape": [ 1536, 1536 ], "size": 245760, "type": 3 }, { "name": "blk.9.attn_q.bias", "offset": 1009157632, "shape": [ 1536 ], "size": 1474560, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1009163776, "shape": [ 1536, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.9.attn_v.bias", "offset": 1010638336, "shape": [ 256 ], "size": 1474560, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1010639360, "shape": [ 1536, 256 ], "size": 1024, "type": 3 }, { "name": "output_norm.weight", "offset": 1010885120, "shape": [ 1536 ], "size": 245760, "type": 0 } ], "version": 3 }