{
    "output_dim": 1024,
    "context_length": 77,
    "vocab_size": 49408,
    "width": 768,
    "heads": 12,
    "layers": 12,
    "xattn": false,
    "fusedLN": true
}