std1.0.0-beta-7 SafeC docs

std :: ml::gpu_cuda_attention

Function std::cuda_flash_attention

#include <std/ml/gpu_cuda_attention.h> source ยท definition

int cuda_flash_attention(
    struct CudaBuf q,
    struct CudaBuf k,
    struct CudaBuf v,
    struct CudaBuf out,
    unsigned long heads,
    unsigned long seqLen,
    unsigned long kvLen,
    unsigned long headDim,
    float scale,
    int causal,
)

out = softmax(q k^T * scale [+ mask]) v. 'scale' <= 0 means 1 / sqrt(headDim). A query with no visible key gets zeros.