std1.0.0-beta-7 SafeC docs

std :: ml::gpu_cuda_attention

Struct std::CudaAttnShape

#include <std/ml/gpu_cuda_attention.h> source

struct CudaAttnShape {
    unsigned long batch;
    unsigned long heads;
    unsigned long kvHeads;
    unsigned long seqLen;
    unsigned long kvLen;
    unsigned long headDim;
    unsigned long window;
}

Fields

§unsigned long batchsource
§unsigned long headssource

query heads

§unsigned long kvHeadssource

key/value heads; divides heads

§unsigned long seqLensource

query length

§unsigned long kvLensource

key/value length

§unsigned long headDimsource

32, 64, 96, 128 or 256

§unsigned long windowsource

flash attention sliding window; 0 = none