GitHub
182.8K
Download
Models
Docs
Pricing
Sign in
Download
Models
Docs
Pricing
Sign in
supergoatscriptguy
/
mnemonic
:126m-q4_0
14
Downloads
Updated
1 week ago
A small chat model written entirely in assembly, trained from scratch on one home graphics card. Outputs are not the best, but considering size and budget they're very good (also sometimes hilarious).
A small chat model written entirely in assembly, trained from scratch on one home graphics card. Outputs are not the best, but considering size and budget they're very good (also sometimes hilarious).
Cancel
126m
300m
mnemonic:126m-q4_0
...
/
model
729b912c1fda · 85MB
Metadata
general.architecture
llama
llama
general.file_type
Q4_0
Q4_0
llama.attention.head_count
12
12
llama.attention.head_count_kv
4
4
llama.attention.layer_norm_rms_epsilon
1e-05
1e-05
llama.block_count
16
16
llama.context_length
1024
1024
llama.embedding_length
768
768
llama.feed_forward_length
2048
2048
llama.rope.dimension_count
64
64
llama.rope.freq_base
10000
10000
llama.vocab_size
32768
32768
tokenizer.ggml.add_bos_token
true
true
tokenizer.ggml.add_eos_token
false
false
tokenizer.ggml.bos_token_id
32752
32752
tokenizer.ggml.eos_token_id
32756
32756
tokenizer.ggml.eot_token_id
32756
32756
tokenizer.ggml.merges
[Ġ t, Ġ a, h e, i n, r e, ...]
[Ġ t, Ġ a, h e, i n, r e, ...]
tokenizer.ggml.model
gpt2
gpt2
tokenizer.ggml.pre
llama-bpe
llama-bpe
tokenizer.ggml.token_type
[1, 1, 1, 1, 1, ...]
[1, 1, 1, 1, 1, ...]
tokenizer.ggml.tokens
[Ā, ā, Ă, ă, Ą, ...]
[Ā, ā, Ă, ă, Ą, ...]
Tensor
Name
Type
Shape
token_embd.weight
Q8_0
Q8_0
[768, 32768]
blk.0
blk.0.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.0.attn_norm.weight
F32
F32
[768]
blk.0.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.0.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.0.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.0.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.0.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.0.ffn_norm.weight
F32
F32
[768]
blk.0.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.1
blk.1.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.1.attn_norm.weight
F32
F32
[768]
blk.1.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.1.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.1.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.1.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.1.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.1.ffn_norm.weight
F32
F32
[768]
blk.1.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.2
blk.2.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.2.attn_norm.weight
F32
F32
[768]
blk.2.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.2.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.2.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.2.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.2.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.2.ffn_norm.weight
F32
F32
[768]
blk.2.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.3
blk.3.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.3.attn_norm.weight
F32
F32
[768]
blk.3.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.3.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.3.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.3.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.3.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.3.ffn_norm.weight
F32
F32
[768]
blk.3.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.4
blk.4.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.4.attn_norm.weight
F32
F32
[768]
blk.4.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.4.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.4.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.4.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.4.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.4.ffn_norm.weight
F32
F32
[768]
blk.4.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.5
blk.5.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.5.attn_norm.weight
F32
F32
[768]
blk.5.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.5.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.5.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.5.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.5.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.5.ffn_norm.weight
F32
F32
[768]
blk.5.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.6
blk.6.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.6.attn_norm.weight
F32
F32
[768]
blk.6.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.6.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.6.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.6.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.6.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.6.ffn_norm.weight
F32
F32
[768]
blk.6.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.7
blk.7.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.7.attn_norm.weight
F32
F32
[768]
blk.7.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.7.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.7.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.7.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.7.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.7.ffn_norm.weight
F32
F32
[768]
blk.7.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.8
blk.8.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.8.attn_norm.weight
F32
F32
[768]
blk.8.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.8.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.8.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.8.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.8.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.8.ffn_norm.weight
F32
F32
[768]
blk.8.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.9
blk.9.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.9.attn_norm.weight
F32
F32
[768]
blk.9.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.9.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.9.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.9.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.9.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.9.ffn_norm.weight
F32
F32
[768]
blk.9.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.10
blk.10.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.10.attn_norm.weight
F32
F32
[768]
blk.10.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.10.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.10.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.10.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.10.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.10.ffn_norm.weight
F32
F32
[768]
blk.10.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.11
blk.11.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.11.attn_norm.weight
F32
F32
[768]
blk.11.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.11.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.11.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.11.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.11.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.11.ffn_norm.weight
F32
F32
[768]
blk.11.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.12
blk.12.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.12.attn_norm.weight
F32
F32
[768]
blk.12.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.12.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.12.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.12.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.12.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.12.ffn_norm.weight
F32
F32
[768]
blk.12.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.13
blk.13.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.13.attn_norm.weight
F32
F32
[768]
blk.13.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.13.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.13.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.13.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.13.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.13.ffn_norm.weight
F32
F32
[768]
blk.13.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.14
blk.14.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.14.attn_norm.weight
F32
F32
[768]
blk.14.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.14.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.14.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.14.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.14.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.14.ffn_norm.weight
F32
F32
[768]
blk.14.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
blk.15
blk.15.attn_k.weight
Q4_0
Q4_0
[768, 256]
blk.15.attn_norm.weight
F32
F32
[768]
blk.15.attn_output.weight
Q4_0
Q4_0
[768, 768]
blk.15.attn_q.weight
Q4_0
Q4_0
[768, 768]
blk.15.attn_v.weight
Q4_0
Q4_0
[768, 256]
blk.15.ffn_down.weight
Q4_0
Q4_0
[2048, 768]
blk.15.ffn_gate.weight
Q4_0
Q4_0
[768, 2048]
blk.15.ffn_norm.weight
F32
F32
[768]
blk.15.ffn_up.weight
Q4_0
Q4_0
[768, 2048]
output_norm.weight
F32
F32
[768]