py-kvcache 2026: Karakterisasi Kinerja External KV Caching vLLM pada NVMe SSD, Asynchronous Direct I/O io_uring, dan Scheduler-Aware Preloading

// py-kvcache High-Performance Offload Connector Architecture (TypeScript Reference Model)
// Berdasarkan Prinsip Arsitektur arXiv:2609.11744 (Kanichai, De Matteis, Trivedi / ETH Zurich, September 2026)
// Mengimplementasikan OffloadingSpec vLLM, Linux io_uring Direct I/O, Bounded Shared Staging,
// Scheduler-Aware Lookahead Preloading, dan Offline Calibrated Break-Even Admission Gating.

export interface BlockHash {
  tokenHash: bigint;
  parentHash: bigint;
  tokenCount: number;
}

export interface OffloadBlockDescriptor {
  blockId: string;
  hashChain: BlockHash;
  storageFilePath: string;
  byteSize: number;       // Contoh: ~28 MiB per 256-token block pada Llama 3.2 3B
  sourceTier: 'GPU_VRAM' | 'CPU_PINNED_STAGING' | 'NVME_SSD';
}

export interface BreakEvenThresholds {
  modelName: string;
  hardwareTier: 'RTX_4000_ADA' | 'NVIDIA_A100' | 'NVIDIA_H100';
  breakEvenTokensSSD: number; // Misal: 6,203 tokens untuk Qwen3-4B pada RTX 4000 Ada
}

export interface PreloadInstruction {
  requestId: string;
  prefixLength: number;
  blocksToStage: OffloadBlockDescriptor[];
  scheduledAdmissionEstimatedMs: number;
}

/**
 * 1. Scheduler-Side Offload Manager
 * Menangani lookup prefix hash, evaluasi break-even Pareto frontier, dan orchestrasi preloading
 */
export class PyKvCacheManager {
  private breakEvenLimits: BreakEvenThresholds;
  private pendingPreloads: Map<string, PreloadInstruction> = new Map();

  constructor(thresholds: BreakEvenThresholds) {
    this.breakEvenLimits = thresholds;
  }

  /**
   * Mengevaluasi apakah pemuatan cache dari SSD NVMe layak secara ekonomis (latency-wise)
   * Dibanding melakukan recomputation komputasi attention penuh pada GPU
   */
  public evaluateAdmissionGate(prefixTokens: number, sourceTier: 'CPU_PINNED_STAGING' | 'NVME_SSD'): {
    shouldLoadFromCache: boolean;
    reason: string;
  } {
    if (sourceTier === 'CPU_PINNED_STAGING') {
      return { shouldLoadFromCache: true, reason: 'CPU pinned memory transfer faster than GPU prefill' };
    }

    // Gating NVMe SSD terhadap ambang batas break-even offline
    if (prefixTokens < this.breakEvenLimits.breakEvenTokensSSD) {
      return {
        shouldLoadFromCache: false,
        reason: `Prefix length (${prefixTokens} tok) < break-even point (${this.breakEvenLimits.breakEvenTokensSSD} tok). Recomputation is faster.`,
      };
    }

    return {
      shouldLoadFromCache: true,
      reason: `Prefix length exceeds break-even threshold. NVMe streaming reduces TTFT.`,
    };
  }

  /**
   * Lookahead Queue Inspection: Mengidentifikasi request yang sedang mengantre (waiting queue)
   * untuk memicu background disk reads sebelum request dialokasikan ke GPU
   */
  public planLookaheadPreload(waitingQueue: Array<{ requestId: string; prefixLength: number; blocks: OffloadBlockDescriptor[] }>): PreloadInstruction[] {
    const dispatchList: PreloadInstruction[] = [];

    for (const req of waitingQueue) {
      const gate = this.evaluateAdmissionGate(req.prefixLength, 'NVME_SSD');
      if (gate.shouldLoadFromCache && !this.pendingPreloads.has(req.requestId)) {
        const instruction: PreloadInstruction = {
          requestId: req.requestId,
          prefixLength: req.prefixLength,
          blocksToStage: req.blocks,
          scheduledAdmissionEstimatedMs: 15.0, // Estimasi waktu tunggu di antrean
        };
        this.pendingPreloads.set(req.requestId, instruction);
        dispatchList.push(instruction);
      }
    }

    return dispatchList;
  }
}

/**
 * 2. Worker-Side Linux io_uring Storage Reactor
 * Mengoperasikan Direct I/O (O_DIRECT) asinkron dengan pembatasan memori CPU staging (bounded depth)
 */
export class IoUringStorageReactor {
  private maxIoDepth: number;
  private currentQueueDepth: number = 0;
  private pinnedStagingPoolSizeBytes: number;
  private allocatedStagingBytes: number = 0;

  constructor(maxIoDepth: number = 16, maxStagingMb: number = 2048) {
    this.maxIoDepth = maxIoDepth;
    this.pinnedStagingPoolSizeBytes = maxStagingMb * 1024 * 1024;
  }

  /**
   * Menyerahkan pekerjaan pembacaan NVMe ke ring buffer io_uring tanpa blocking thread
   */
  public async submitAsynchronousDirectRead(block: OffloadBlockDescriptor): Promise<{ blockId: string; stagingAddress: bigint; durationMs: number }> {
    if (this.currentQueueDepth >= this.maxIoDepth) {
      throw new Error('io_uring submission queue full: backpressure applied to bound memory');
    }

    if (this.allocatedStagingBytes + block.byteSize > this.pinnedStagingPoolSizeBytes) {
      throw new Error('CPU Staging Pool exhausted: rejecting preload to prevent system OOM');
    }

    this.currentQueueDepth++;
    this.allocatedStagingBytes += block.byteSize;
    const start = performance.now();

    try {
      // Simulasi eksekusi io_uring SQE (Submission Queue Entry) dengan direct I/O bypass
      // Membaca file blok teragregasi (~28 MiB) langsung ke pinned host memory
      await new Promise(resolve => setTimeout(resolve, Math.max(1, Math.round(block.byteSize / (3.5 * 1024 * 1024))))); // Simulasi ~3.5 GB/s transfer NVMe
      
      const elapsed = performance.now() - start;
      return {
        blockId: block.blockId,
        stagingAddress: BigInt('0x7f00deadbeef'),
        durationMs: Number(elapsed.toFixed(2)),
      };
    } finally {
      this.currentQueueDepth--;
    }
  }

  public releaseStagingBuffer(bytes: number): void {
    this.allocatedStagingBytes = Math.max(0, this.allocatedStagingBytes - bytes);
  }
}