CUA-Universe 2026: Arsitektur App-Forge, Task-Weave, dan Path-Steer untuk Hybrid GUI+CLI Autonomous Agents

// CUA-Universe: Hybrid GUI+CLI Agent Execution Kernel & Path-Steer Engine
// Spesifikasi Berdasarkan Riset arXiv:2609.05374 (September 2026)
// Mengorkestrasi Aksi Multimodal atas Shared Persistent VM Application State

export type ActionModality = 'gui' | 'cli';

export interface GuiActionPayload {
  type: 'click' | 'double_click' | 'drag_and_drop' | 'type_text' | 'hotkey' | 'scroll';
  coordinates?: { x: number; y: number };
  targetCoordinates?: { x: number; y: number };
  text?: string;
  keyCombination?: string[];
}

export interface CliActionPayload {
  toolName: string;
  command: string;
  targetFilePath?: string;
  timeoutMs?: number;
}

export interface HybridAgentAction {
  id: string;
  stepIndex: number;
  modality: ActionModality;
  guiAction?: GuiActionPayload;
  cliAction?: CliActionPayload;
  reasoningRationale: string;
}

export interface MultimodalObservation {
  stepIndex: number;
  screenshotBase64: string; // Visual perception (v_t)
  cliStdout?: string;        // Textual CLI execution return (c_t)
  cliStderr?: string;
  exitCode?: number;
  activeWindowMeta: {
    appId: string;
    windowTitle: string;
    focusedElementId?: string;
  };
}

export interface OperationAbstraction {
  verb: string;                  // e.g., 'export_scene_to_gltf', 'align_layers_vertical'
  preferredModality: ActionModality;
  cliTemplate?: string;          // e.g., 'blender --background {file} --python-expr "..."'
  expectedTokenCostGui: number;  // Estimasi token screenshot + visual inference
  expectedTokenCostCli: number;  // Estimasi token textual prompt + command
  stateResyncRequired: boolean;  // Memerlukan refresh GUI setelah eksekusi background CLI
}

export class CUAUniverseKernel {
  private operationRegistry = new Map();
  private activeTrajectory: HybridAgentAction[] = [];
  private stepCount = 0;
  private totalTokensUsed = 0;

  constructor() {
    this.registerCanonicalOperations();
  }

  /**
   * Pendaftaran operasi dasar dari App-Forge & Task-Weave
   */
  private registerCanonicalOperations() {
    this.operationRegistry.set('export_3d_asset', {
      verb: 'export_scene_to_gltf',
      preferredModality: 'cli',
      cliTemplate: 'blender -b "{projectPath}" --python-expr "import bpy; bpy.ops.export_scene.gltf(filepath=\"{outputPath}\")"',
      expectedTokenCostGui: 4800, // Menavigasi menu File -> Export -> Dialog memakan 4-6 step GUI (~4.8k tokens)
      expectedTokenCostCli: 450,  // Eksekusi terminal tunggal (~450 tokens)
      stateResyncRequired: false,
    });

    this.operationRegistry.set('inspect_layout_alignment', {
      verb: 'verify_visual_alignment',
      preferredModality: 'gui',
      expectedTokenCostGui: 1200, // Visual inspection lewat screenshot VLM
      expectedTokenCostCli: 8500, // Menghitung bounding box geometris via headless script sangat mahal
      stateResyncRequired: false,
    });

    this.operationRegistry.set('batch_replace_attributes', {
      verb: 'batch_modify_xml_tags',
      preferredModality: 'cli',
      cliTemplate: 'sed -i "s/{oldVal}/{newVal}/g" "{targetFile}"',
      expectedTokenCostGui: 16400, // 20+ klik mouse manual
      expectedTokenCostCli: 380,   // Satu baris regex sed
      stateResyncRequired: true,   // Membutuhkan reload file di GUI canvas
    });
  }

  /**
   * Path-Steer: Efficiency-Aware Modality Router
   * Memilih antara GUI dan CLI berdasarkan keunggulan komparatif modalitas
   */
  public routeNextAction(
    targetOperation: string,
    observation: MultimodalObservation
  ): { modality: ActionModality; rationale: string } {
    const op = this.operationRegistry.get(targetOperation);

    if (!op) {
      // Default ke GUI bila operasi tidak memiliki abstraksi CLI terdaftar
      return {
        modality: 'gui',
        rationale: 'Operasi baru/tidak dikenal: Menggunakan inspeksi GUI langsung.',
      };
    }

    // Evaluasi rasio efisiensi token & reliabilitas deterministik
    if (op.preferredModality === 'cli' && op.cliTemplate) {
      const tokenSavings = op.expectedTokenCostGui - op.expectedTokenCostCli;
      return {
        modality: 'cli',
        rationale: `Path-Steer memilih CLI: Menghemat ${tokenSavings} tokens dan mengeliminasi 3-8 step klik serial GUI.`,
      };
    }

    return {
      modality: 'gui',
      rationale: 'Path-Steer memilih GUI: Tugas membutuhkan penalaran persepsi spasial/layout visual yang tidak dapat diakses via API deterministik.',
    };
  }

  /**
   * Eksekusi Aksi Hybrid & Sinkronisasi State POMDP
   */
  public async executeHybridStep(
    action: HybridAgentAction,
    vmBridge: {
      sendGuiInput: (payload: GuiActionPayload) => Promise;
      sendCliCommand: (cmd: string) => Promise<{ stdout: string; stderr: string; code: number }>;
      resyncGuiView: (appId: string) => Promise;
      captureScreen: () => Promise;
    }
  ): Promise {
    this.stepCount++;
    this.activeTrajectory.push(action);

    let cliStdout: string | undefined;
    let cliStderr: string | undefined;
    let exitCode: number | undefined;

    if (action.modality === 'cli' && action.cliAction) {
      const res = await vmBridge.sendCliCommand(action.cliAction.command);
      cliStdout = res.stdout;
      cliStderr = res.stderr;
      exitCode = res.code;
      this.totalTokensUsed += 350; // Estimasi token teks CLI

      // Jika operasi mengubah state disk lokal secara eksternal, picu adapter GUI resync
      const op = Array.from(this.operationRegistry.values()).find(
        (o) => o.cliTemplate && action.cliAction?.command.includes(o.verb)
      );
      if (op?.stateResyncRequired) {
        await vmBridge.resyncGuiView('active_app');
      }
    } else if (action.modality === 'gui' && action.guiAction) {
      await vmBridge.sendGuiInput(action.guiAction);
      this.totalTokensUsed += 1450; // Estimasi token screenshot + visual coordinates
    }

    // Ambil observasi baru
    const screenshot = await vmBridge.captureScreen();

    return {
      stepIndex: this.stepCount,
      screenshotBase64: screenshot,
      cliStdout,
      cliStderr,
      exitCode,
      activeWindowMeta: {
        appId: 'active_app',
        windowTitle: 'Main Workspace',
      },
    };
  }

  /**
   * Trajectory Harvesting & Efficiency Metrics
   */
  public evaluateTrajectoryEfficiency(verifierScore: number): {
    accepted: boolean;
    totalSteps: number;
    totalTokens: number;
    efficiencyGain: number; // Rasio efisiensi dibanding baseline GUI-only
  } {
    // Kriteria Path-Steer: trajektori diterima jika skor verifier >= 0.75
    const accepted = verifierScore >= 0.75;
    const baselineEstimatedTokens = this.stepCount * 1450 * 1.6; // Baseline GUI murni
    const efficiencyGain = Number((baselineEstimatedTokens / Math.max(1, this.totalTokensUsed)).toFixed(2));

    return {
      accepted,
      totalSteps: this.stepCount,
      totalTokens: this.totalTokensUsed,
      efficiencyGain,
    };
  }
}