Struct std::CudaAttnShape
struct CudaAttnShape {
unsigned long batch;
unsigned long heads;
unsigned long kvHeads;
unsigned long seqLen;
unsigned long kvLen;
unsigned long headDim;
unsigned long window;
}Fields
query heads
key/value heads; divides heads
query length
key/value length
32, 64, 96, 128 or 256
flash attention sliding window; 0 = none