jersobh/phi-law-ollama/model

jersobh/ phi-law-ollama:latest

3 Downloads Updated 1 year ago

phi-law-ollama:latest ... /

model

13f4db119ae0 · 445MB

Metadata

general.architecture

phi3

phi3
general.file_type

F16

F16
phi3.attention.head_count

32

32
phi3.attention.head_count_kv

32

32
phi3.attention.layer_norm_rms_epsilon

1e-05

1e-05
phi3.attention.sliding_window

2047

2047
phi3.block_count

32

32
phi3.context_length

4096

4096
phi3.embedding_length

3072

3072
phi3.feed_forward_length

8192

8192
phi3.rope.dimension_count

96

96
phi3.rope.freq_base

10000

10000
phi3.rope.scaling.original_context_length

4096

4096
tokenizer.ggml.add_bos_token

false

false
tokenizer.ggml.add_eos_token

false

false
tokenizer.ggml.add_padding_token

false

false
tokenizer.ggml.add_unknown_token

false

false
tokenizer.ggml.bos_token_id

1

1
tokenizer.ggml.eos_token_id

32000

32000
tokenizer.ggml.merges

[▁ t, e r, i n, ▁ a, e n, ...]

[▁ t, e r, i n, ▁ a, e n, ...]
tokenizer.ggml.model

llama

llama
tokenizer.ggml.padding_token_id

32000

32000
tokenizer.ggml.pre

default

default
tokenizer.ggml.scores

[0, 0, 0, 0, 0, ...]

[0, 0, 0, 0, 0, ...]
tokenizer.ggml.token_type

[2, 3, 3, 6, 6, ...]

[2, 3, 3, 6, 6, ...]
tokenizer.ggml.tokens

[<unk>, <s>, </s>, <0x00>, <0x01>, ...]

[<unk>, <s>, </s>, <0x00>, <0x01>, ...]
tokenizer.ggml.unknown_token_id

0

0

Tensor

Name

Type

Shape

base_model.model.blk.0

base_model.model.blk.0.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.0.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.0.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.0.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.0.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.0.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.0.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.0.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.1

base_model.model.blk.1.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.1.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.1.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.1.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.1.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.1.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.1.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.1.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.2

base_model.model.blk.2.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.2.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.2.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.2.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.2.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.2.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.2.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.2.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.3

base_model.model.blk.3.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.3.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.3.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.3.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.3.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.3.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.3.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.3.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.4

base_model.model.blk.4.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.4.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.4.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.4.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.4.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.4.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.4.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.4.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.5

base_model.model.blk.5.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.5.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.5.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.5.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.5.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.5.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.5.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.5.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.6

base_model.model.blk.6.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.6.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.6.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.6.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.6.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.6.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.6.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.6.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.7

base_model.model.blk.7.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.7.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.7.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.7.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.7.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.7.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.7.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.7.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.8

base_model.model.blk.8.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.8.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.8.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.8.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.8.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.8.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.8.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.8.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.9

base_model.model.blk.9.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.9.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.9.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.9.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.9.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.9.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.9.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.9.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.10

base_model.model.blk.10.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.10.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.10.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.10.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.10.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.10.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.10.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.10.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.11

base_model.model.blk.11.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.11.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.11.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.11.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.11.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.11.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.11.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.11.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.12

base_model.model.blk.12.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.12.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.12.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.12.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.12.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.12.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.12.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.12.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.13

base_model.model.blk.13.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.13.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.13.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.13.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.13.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.13.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.13.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.13.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.14

base_model.model.blk.14.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.14.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.14.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.14.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.14.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.14.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.14.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.14.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.15

base_model.model.blk.15.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.15.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.15.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.15.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.15.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.15.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.15.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.15.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.16

base_model.model.blk.16.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.16.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.16.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.16.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.16.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.16.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.16.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.16.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.17

base_model.model.blk.17.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.17.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.17.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.17.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.17.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.17.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.17.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.17.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.18

base_model.model.blk.18.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.18.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.18.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.18.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.18.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.18.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.18.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.18.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.19

base_model.model.blk.19.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.19.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.19.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.19.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.19.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.19.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.19.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.19.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.20

base_model.model.blk.20.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.20.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.20.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.20.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.20.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.20.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.20.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.20.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.21

base_model.model.blk.21.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.21.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.21.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.21.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.21.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.21.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.21.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.21.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.22

base_model.model.blk.22.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.22.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.22.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.22.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.22.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.22.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.22.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.22.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.23

base_model.model.blk.23.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.23.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.23.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.23.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.23.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.23.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.23.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.23.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.24

base_model.model.blk.24.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.24.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.24.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.24.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.24.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.24.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.24.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.24.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.25

base_model.model.blk.25.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.25.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.25.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.25.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.25.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.25.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.25.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.25.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.26

base_model.model.blk.26.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.26.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.26.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.26.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.26.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.26.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.26.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.26.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.27

base_model.model.blk.27.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.27.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.27.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.27.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.27.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.27.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.27.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.27.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.28

base_model.model.blk.28.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.28.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.28.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.28.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.28.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.28.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.28.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.28.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.29

base_model.model.blk.29.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.29.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.29.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.29.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.29.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.29.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.29.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.29.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.30

base_model.model.blk.30.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.30.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.30.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.30.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.30.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.30.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.30.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.30.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.blk.31

base_model.model.blk.31.attn_output.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.31.attn_output.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.31.attn_qkv.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.31.attn_qkv.lora_B.weight

F16

F16

[16, 9216]
base_model.model.blk.31.ffn_down.lora_A.weight

F16

F16

[8192, 16]
base_model.model.blk.31.ffn_down.lora_B.weight

F16

F16

[16, 3072]
base_model.model.blk.31.ffn_up.lora_A.weight

F16

F16

[3072, 16]
base_model.model.blk.31.ffn_up.lora_B.weight

F16

F16

[16, 16384]

base_model.model.output.weight

F16

F16

[3072, 32011]
base_model.model.token_embd.weight

F16

F16

[3072, 32011]