supergoatscriptguy/ mnemonic:300m-f16

14 1 week ago

A small chat model written entirely in assembly, trained from scratch on one home graphics card. Outputs are not the best, but considering size and budget they're very good (also sometimes hilarious).

126m 300m
... /
model
ad7da59fa1ee · 610MB
    Metadata
  • general.architecture
    llama
  • general.file_type
    F16
  • llama.attention.head_count
    16
  • llama.attention.head_count_kv
    4
  • llama.attention.layer_norm_rms_epsilon
    1e-05
  • llama.block_count
    24
  • llama.context_length
    2048
  • llama.embedding_length
    1024
  • llama.feed_forward_length
    2816
  • llama.rope.dimension_count
    64
  • llama.rope.freq_base
    10000
  • llama.vocab_size
    32768
  • tokenizer.ggml.add_bos_token
    true
  • tokenizer.ggml.add_eos_token
    false
  • tokenizer.ggml.bos_token_id
    32752
  • tokenizer.ggml.eos_token_id
    32756
  • tokenizer.ggml.eot_token_id
    32756
  • tokenizer.ggml.merges
    [Ġ t, Ġ a, h e, i n, r e, ...]
  • tokenizer.ggml.model
    gpt2
  • tokenizer.ggml.pre
    llama-bpe
  • tokenizer.ggml.token_type
    [1, 1, 1, 1, 1, ...]
  • tokenizer.ggml.tokens
    [Ā, ā, Ă, ă, Ą, ...]
    • Tensor
  • token_embd.weight
    F16
    [1024, 32768]
    • blk.0
  • blk.0.attn_k.weight
    F16
    [1024, 256]
  • blk.0.attn_norm.weight
    F32
    [1024]
  • blk.0.attn_output.weight
    F16
    [1024, 1024]
  • blk.0.attn_q.weight
    F16
    [1024, 1024]
  • blk.0.attn_v.weight
    F16
    [1024, 256]
  • blk.0.ffn_down.weight
    F16
    [2816, 1024]
  • blk.0.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.0.ffn_norm.weight
    F32
    [1024]
  • blk.0.ffn_up.weight
    F16
    [1024, 2816]
    • blk.1
  • blk.1.attn_k.weight
    F16
    [1024, 256]
  • blk.1.attn_norm.weight
    F32
    [1024]
  • blk.1.attn_output.weight
    F16
    [1024, 1024]
  • blk.1.attn_q.weight
    F16
    [1024, 1024]
  • blk.1.attn_v.weight
    F16
    [1024, 256]
  • blk.1.ffn_down.weight
    F16
    [2816, 1024]
  • blk.1.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.1.ffn_norm.weight
    F32
    [1024]
  • blk.1.ffn_up.weight
    F16
    [1024, 2816]
    • blk.2
  • blk.2.attn_k.weight
    F16
    [1024, 256]
  • blk.2.attn_norm.weight
    F32
    [1024]
  • blk.2.attn_output.weight
    F16
    [1024, 1024]
  • blk.2.attn_q.weight
    F16
    [1024, 1024]
  • blk.2.attn_v.weight
    F16
    [1024, 256]
  • blk.2.ffn_down.weight
    F16
    [2816, 1024]
  • blk.2.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.2.ffn_norm.weight
    F32
    [1024]
  • blk.2.ffn_up.weight
    F16
    [1024, 2816]
    • blk.3
  • blk.3.attn_k.weight
    F16
    [1024, 256]
  • blk.3.attn_norm.weight
    F32
    [1024]
  • blk.3.attn_output.weight
    F16
    [1024, 1024]
  • blk.3.attn_q.weight
    F16
    [1024, 1024]
  • blk.3.attn_v.weight
    F16
    [1024, 256]
  • blk.3.ffn_down.weight
    F16
    [2816, 1024]
  • blk.3.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.3.ffn_norm.weight
    F32
    [1024]
  • blk.3.ffn_up.weight
    F16
    [1024, 2816]
    • blk.4
  • blk.4.attn_k.weight
    F16
    [1024, 256]
  • blk.4.attn_norm.weight
    F32
    [1024]
  • blk.4.attn_output.weight
    F16
    [1024, 1024]
  • blk.4.attn_q.weight
    F16
    [1024, 1024]
  • blk.4.attn_v.weight
    F16
    [1024, 256]
  • blk.4.ffn_down.weight
    F16
    [2816, 1024]
  • blk.4.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.4.ffn_norm.weight
    F32
    [1024]
  • blk.4.ffn_up.weight
    F16
    [1024, 2816]
    • blk.5
  • blk.5.attn_k.weight
    F16
    [1024, 256]
  • blk.5.attn_norm.weight
    F32
    [1024]
  • blk.5.attn_output.weight
    F16
    [1024, 1024]
  • blk.5.attn_q.weight
    F16
    [1024, 1024]
  • blk.5.attn_v.weight
    F16
    [1024, 256]
  • blk.5.ffn_down.weight
    F16
    [2816, 1024]
  • blk.5.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.5.ffn_norm.weight
    F32
    [1024]
  • blk.5.ffn_up.weight
    F16
    [1024, 2816]
    • blk.6
  • blk.6.attn_k.weight
    F16
    [1024, 256]
  • blk.6.attn_norm.weight
    F32
    [1024]
  • blk.6.attn_output.weight
    F16
    [1024, 1024]
  • blk.6.attn_q.weight
    F16
    [1024, 1024]
  • blk.6.attn_v.weight
    F16
    [1024, 256]
  • blk.6.ffn_down.weight
    F16
    [2816, 1024]
  • blk.6.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.6.ffn_norm.weight
    F32
    [1024]
  • blk.6.ffn_up.weight
    F16
    [1024, 2816]
    • blk.7
  • blk.7.attn_k.weight
    F16
    [1024, 256]
  • blk.7.attn_norm.weight
    F32
    [1024]
  • blk.7.attn_output.weight
    F16
    [1024, 1024]
  • blk.7.attn_q.weight
    F16
    [1024, 1024]
  • blk.7.attn_v.weight
    F16
    [1024, 256]
  • blk.7.ffn_down.weight
    F16
    [2816, 1024]
  • blk.7.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.7.ffn_norm.weight
    F32
    [1024]
  • blk.7.ffn_up.weight
    F16
    [1024, 2816]
    • blk.8
  • blk.8.attn_k.weight
    F16
    [1024, 256]
  • blk.8.attn_norm.weight
    F32
    [1024]
  • blk.8.attn_output.weight
    F16
    [1024, 1024]
  • blk.8.attn_q.weight
    F16
    [1024, 1024]
  • blk.8.attn_v.weight
    F16
    [1024, 256]
  • blk.8.ffn_down.weight
    F16
    [2816, 1024]
  • blk.8.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.8.ffn_norm.weight
    F32
    [1024]
  • blk.8.ffn_up.weight
    F16
    [1024, 2816]
    • blk.9
  • blk.9.attn_k.weight
    F16
    [1024, 256]
  • blk.9.attn_norm.weight
    F32
    [1024]
  • blk.9.attn_output.weight
    F16
    [1024, 1024]
  • blk.9.attn_q.weight
    F16
    [1024, 1024]
  • blk.9.attn_v.weight
    F16
    [1024, 256]
  • blk.9.ffn_down.weight
    F16
    [2816, 1024]
  • blk.9.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.9.ffn_norm.weight
    F32
    [1024]
  • blk.9.ffn_up.weight
    F16
    [1024, 2816]
    • blk.10
  • blk.10.attn_k.weight
    F16
    [1024, 256]
  • blk.10.attn_norm.weight
    F32
    [1024]
  • blk.10.attn_output.weight
    F16
    [1024, 1024]
  • blk.10.attn_q.weight
    F16
    [1024, 1024]
  • blk.10.attn_v.weight
    F16
    [1024, 256]
  • blk.10.ffn_down.weight
    F16
    [2816, 1024]
  • blk.10.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.10.ffn_norm.weight
    F32
    [1024]
  • blk.10.ffn_up.weight
    F16
    [1024, 2816]
    • blk.11
  • blk.11.attn_k.weight
    F16
    [1024, 256]
  • blk.11.attn_norm.weight
    F32
    [1024]
  • blk.11.attn_output.weight
    F16
    [1024, 1024]
  • blk.11.attn_q.weight
    F16
    [1024, 1024]
  • blk.11.attn_v.weight
    F16
    [1024, 256]
  • blk.11.ffn_down.weight
    F16
    [2816, 1024]
  • blk.11.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.11.ffn_norm.weight
    F32
    [1024]
  • blk.11.ffn_up.weight
    F16
    [1024, 2816]
    • blk.12
  • blk.12.attn_k.weight
    F16
    [1024, 256]
  • blk.12.attn_norm.weight
    F32
    [1024]
  • blk.12.attn_output.weight
    F16
    [1024, 1024]
  • blk.12.attn_q.weight
    F16
    [1024, 1024]
  • blk.12.attn_v.weight
    F16
    [1024, 256]
  • blk.12.ffn_down.weight
    F16
    [2816, 1024]
  • blk.12.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.12.ffn_norm.weight
    F32
    [1024]
  • blk.12.ffn_up.weight
    F16
    [1024, 2816]
    • blk.13
  • blk.13.attn_k.weight
    F16
    [1024, 256]
  • blk.13.attn_norm.weight
    F32
    [1024]
  • blk.13.attn_output.weight
    F16
    [1024, 1024]
  • blk.13.attn_q.weight
    F16
    [1024, 1024]
  • blk.13.attn_v.weight
    F16
    [1024, 256]
  • blk.13.ffn_down.weight
    F16
    [2816, 1024]
  • blk.13.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.13.ffn_norm.weight
    F32
    [1024]
  • blk.13.ffn_up.weight
    F16
    [1024, 2816]
    • blk.14
  • blk.14.attn_k.weight
    F16
    [1024, 256]
  • blk.14.attn_norm.weight
    F32
    [1024]
  • blk.14.attn_output.weight
    F16
    [1024, 1024]
  • blk.14.attn_q.weight
    F16
    [1024, 1024]
  • blk.14.attn_v.weight
    F16
    [1024, 256]
  • blk.14.ffn_down.weight
    F16
    [2816, 1024]
  • blk.14.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.14.ffn_norm.weight
    F32
    [1024]
  • blk.14.ffn_up.weight
    F16
    [1024, 2816]
    • blk.15
  • blk.15.attn_k.weight
    F16
    [1024, 256]
  • blk.15.attn_norm.weight
    F32
    [1024]
  • blk.15.attn_output.weight
    F16
    [1024, 1024]
  • blk.15.attn_q.weight
    F16
    [1024, 1024]
  • blk.15.attn_v.weight
    F16
    [1024, 256]
  • blk.15.ffn_down.weight
    F16
    [2816, 1024]
  • blk.15.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.15.ffn_norm.weight
    F32
    [1024]
  • blk.15.ffn_up.weight
    F16
    [1024, 2816]
    • blk.16
  • blk.16.attn_k.weight
    F16
    [1024, 256]
  • blk.16.attn_norm.weight
    F32
    [1024]
  • blk.16.attn_output.weight
    F16
    [1024, 1024]
  • blk.16.attn_q.weight
    F16
    [1024, 1024]
  • blk.16.attn_v.weight
    F16
    [1024, 256]
  • blk.16.ffn_down.weight
    F16
    [2816, 1024]
  • blk.16.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.16.ffn_norm.weight
    F32
    [1024]
  • blk.16.ffn_up.weight
    F16
    [1024, 2816]
    • blk.17
  • blk.17.attn_k.weight
    F16
    [1024, 256]
  • blk.17.attn_norm.weight
    F32
    [1024]
  • blk.17.attn_output.weight
    F16
    [1024, 1024]
  • blk.17.attn_q.weight
    F16
    [1024, 1024]
  • blk.17.attn_v.weight
    F16
    [1024, 256]
  • blk.17.ffn_down.weight
    F16
    [2816, 1024]
  • blk.17.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.17.ffn_norm.weight
    F32
    [1024]
  • blk.17.ffn_up.weight
    F16
    [1024, 2816]
    • blk.18
  • blk.18.attn_k.weight
    F16
    [1024, 256]
  • blk.18.attn_norm.weight
    F32
    [1024]
  • blk.18.attn_output.weight
    F16
    [1024, 1024]
  • blk.18.attn_q.weight
    F16
    [1024, 1024]
  • blk.18.attn_v.weight
    F16
    [1024, 256]
  • blk.18.ffn_down.weight
    F16
    [2816, 1024]
  • blk.18.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.18.ffn_norm.weight
    F32
    [1024]
  • blk.18.ffn_up.weight
    F16
    [1024, 2816]
    • blk.19
  • blk.19.attn_k.weight
    F16
    [1024, 256]
  • blk.19.attn_norm.weight
    F32
    [1024]
  • blk.19.attn_output.weight
    F16
    [1024, 1024]
  • blk.19.attn_q.weight
    F16
    [1024, 1024]
  • blk.19.attn_v.weight
    F16
    [1024, 256]
  • blk.19.ffn_down.weight
    F16
    [2816, 1024]
  • blk.19.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.19.ffn_norm.weight
    F32
    [1024]
  • blk.19.ffn_up.weight
    F16
    [1024, 2816]
    • blk.20
  • blk.20.attn_k.weight
    F16
    [1024, 256]
  • blk.20.attn_norm.weight
    F32
    [1024]
  • blk.20.attn_output.weight
    F16
    [1024, 1024]
  • blk.20.attn_q.weight
    F16
    [1024, 1024]
  • blk.20.attn_v.weight
    F16
    [1024, 256]
  • blk.20.ffn_down.weight
    F16
    [2816, 1024]
  • blk.20.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.20.ffn_norm.weight
    F32
    [1024]
  • blk.20.ffn_up.weight
    F16
    [1024, 2816]
    • blk.21
  • blk.21.attn_k.weight
    F16
    [1024, 256]
  • blk.21.attn_norm.weight
    F32
    [1024]
  • blk.21.attn_output.weight
    F16
    [1024, 1024]
  • blk.21.attn_q.weight
    F16
    [1024, 1024]
  • blk.21.attn_v.weight
    F16
    [1024, 256]
  • blk.21.ffn_down.weight
    F16
    [2816, 1024]
  • blk.21.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.21.ffn_norm.weight
    F32
    [1024]
  • blk.21.ffn_up.weight
    F16
    [1024, 2816]
    • blk.22
  • blk.22.attn_k.weight
    F16
    [1024, 256]
  • blk.22.attn_norm.weight
    F32
    [1024]
  • blk.22.attn_output.weight
    F16
    [1024, 1024]
  • blk.22.attn_q.weight
    F16
    [1024, 1024]
  • blk.22.attn_v.weight
    F16
    [1024, 256]
  • blk.22.ffn_down.weight
    F16
    [2816, 1024]
  • blk.22.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.22.ffn_norm.weight
    F32
    [1024]
  • blk.22.ffn_up.weight
    F16
    [1024, 2816]
    • blk.23
  • blk.23.attn_k.weight
    F16
    [1024, 256]
  • blk.23.attn_norm.weight
    F32
    [1024]
  • blk.23.attn_output.weight
    F16
    [1024, 1024]
  • blk.23.attn_q.weight
    F16
    [1024, 1024]
  • blk.23.attn_v.weight
    F16
    [1024, 256]
  • blk.23.ffn_down.weight
    F16
    [2816, 1024]
  • blk.23.ffn_gate.weight
    F16
    [1024, 2816]
  • blk.23.ffn_norm.weight
    F32
    [1024]
  • blk.23.ffn_up.weight
    F16
    [1024, 2816]
  • output_norm.weight
    F32
    [1024]