{
    "output_dim": 1024,
    "context_length": 77,
    "vocab_size": 49408,
    "width": 1024,
    "heads": 16,
    "layers": 24,
    "xattn": true,
    "fusedLN": true
}