X-Tree 2026: Arsitektur Tokenisasi Pengalaman Agen Otonom, X-Score Recursive Merging, dan Pelatihan Hierarki Skill ke Bobot Model Tanpa Ekstraksi LLM

// Production Implementation: X-Tree Tokenizer, X-Score Recursive Merger & Multi-Mode Training Controller
// Berdasarkan Arsitektur Waterloo / Duke / NUS (arXiv:2609.32993 - Sitao Cheng et al., 2026)
//
// Karakteristik Inti:
// 1. Deterministic Tokenization: Zero LLM calls via X-Score = f_uv * (l_u + l_v)^p_l * (succ(uv) + eps)^p_s
// 2. Information-Theoretic Compression Gate: f_uv - 1 > eta * (l_u + l_v)
// 3. Offline RL Node Sampling: Depth-scaled node completion reward r = (1-alpha)*match_ratio + alpha*(1+gamma*d_v)*c_v
// 4. Online RLVR Adaptive Skill Bonus: r_i = R_out + lambda_g * sum(b_v) dengan annealing dinamis saat verifier aktif
// 5. OPSD Distillation Supervisor: Confidence-gated token distillation g_t = sigma(beta * delta_t)

export interface PrimitiveAction {
  id: string;
  actionType: 'click' | 'fill' | 'navigate' | 'select' | 'submit';
  targetElement: string;
  payload?: string;
  isCanonical: boolean;
}

export interface XTreeNode {
  nodeId: string;
  symbol: string;
  leftChild?: XTreeNode;
  rightChild?: XTreeNode;
  primitiveSequence: PrimitiveAction[];
  length: number;       // l_v: jumlah aksi primitif
  depth: number;        // d_v: kedalaman hierarki pohon
  frequency: number;    // f_uv: frekuensi kemunculan
  successRate: number;  // succ(uv): rasio kemunculan di episode sukses
  xScore: number;       // Skor merger akhir
}

export interface CandidateMerge {
  leftSymbol: string;
  rightSymbol: string;
  combinedSymbol: string;
  frequency: number;
  totalLength: number;
  successRate: number;
  xScore: number;
}

export class XTreeTokenizer {
  private vocabulary: Map = new Map();
  private compressionPenaltyEta: number = 0.5;
  private pLength: number = 1.0;
  private pSuccess: number = 1.0;
  private epsilonSmoothing: number = 0.05;

  constructor(options?: { eta?: number; pL?: number; pS?: number; eps?: number }) {
    if (options?.eta !== undefined) this.compressionPenaltyEta = options.eta;
    if (options?.pL !== undefined) this.pLength = options.pL;
    if (options?.pS !== undefined) this.pSuccess = options.pS;
    if (options?.eps !== undefined) this.epsilonSmoothing = options.eps;
  }

  /**
   * Menghitung X-Score untuk pasangan simbol adjasen:
   * X(u,v) = f_uv * (l_u + l_v)^p_l * (succ(uv) + eps)^p_s
   */
  public calculateXScore(frequency: number, totalLength: number, successRate: number): number {
    const recurrenceTerm = frequency;
    const lengthTerm = Math.pow(totalLength, this.pLength);
    const successTerm = Math.pow(successRate + this.epsilonSmoothing, this.pSuccess);
    return recurrenceTerm * lengthTerm * successTerm;
  }

  /**
   * Memeriksa gerbang kompresi: f_uv - 1 > eta * (l_u + l_v)
   */
  public satisfiesCompressionGate(frequency: number, totalLength: number): boolean {
    return (frequency - 1) > (this.compressionPenaltyEta * totalLength);
  }

  /**
   * Melakukan recursive merging deterministik tanpa panggilan LLM
   */
  public mineXTree(
    trajectories: Array<{ actions: PrimitiveAction[]; isSuccessful: boolean }>,
    maxMerges: number = 256
  ): Map {
    // 1. Inisialisasi kosakata dengan aksi primitif kanonikal (l=1, depth=0)
    for (const traj of trajectories) {
      for (const act of traj.actions) {
        const canonicalKey = `${act.actionType}<${act.targetElement}>`;
        if (!this.vocabulary.has(canonicalKey)) {
          this.vocabulary.set(canonicalKey, {
            nodeId: `node-${nanoid(6)}`,
            symbol: canonicalKey,
            primitiveSequence: [act],
            length: 1,
            depth: 0,
            frequency: 1,
            successRate: traj.isSuccessful ? 1.0 : 0.0,
            xScore: 0,
          });
        }
      }
    }

    // 2. Iterative Pair Mining Loop
    let mergeCount = 0;
    while (mergeCount < maxMerges) {
      const pairStats = new Map();

      // Scanning n-gram pasangan bersebelahan di seluruh korpus
      for (const traj of trajectories) {
        const symbols = traj.actions.map(a => `${a.actionType}<${a.targetElement}>`);
        for (let i = 0; i < symbols.length - 1; i++) {
          const u = symbols[i];
          const v = symbols[i + 1];
          const pairKey = `${u}__+__${v}`;
          const entry = pairStats.get(pairKey) || { count: 0, successCount: 0, left: u, right: v };
          entry.count += 1;
          if (traj.isSuccessful) entry.successCount += 1;
          pairStats.set(pairKey, entry);
        }
      }

      // Evaluasi kandidat merger terbaik berdasarkan X-Score
      let bestCandidate: CandidateMerge | null = null;
      for (const [key, stats] of pairStats.entries()) {
        const leftNode = this.vocabulary.get(stats.left);
        const rightNode = this.vocabulary.get(stats.right);
        if (!leftNode || !rightNode) continue;

        const totalLength = leftNode.length + rightNode.length;
        if (!this.satisfiesCompressionGate(stats.count, totalLength)) continue;

        const successRate = stats.count > 0 ? stats.successCount / stats.count : 0;
        const score = this.calculateXScore(stats.count, totalLength, successRate);

        if (!bestCandidate || score > bestCandidate.xScore) {
          bestCandidate = {
            leftSymbol: stats.left,
            rightSymbol: stats.right,
            combinedSymbol: `(${stats.left} -> ${stats.right})`,
            frequency: stats.count,
            totalLength,
            successRate,
            xScore: score,
          };
        }
      }

      // Berhenti jika tidak ada pasangan yang memenuhi kriteria kompresi
      if (!bestCandidate) break;

      // Bentuk node hierarki baru
      const leftChild = this.vocabulary.get(bestCandidate.leftSymbol)!;
      const rightChild = this.vocabulary.get(bestCandidate.rightSymbol)!;
      const newNode: XTreeNode = {
        nodeId: `xtree-${nanoid(6)}`,
        symbol: bestCandidate.combinedSymbol,
        leftChild,
        rightChild,
        primitiveSequence: [...leftChild.primitiveSequence, ...rightChild.primitiveSequence],
        length: bestCandidate.totalLength,
        depth: 1 + Math.max(leftChild.depth, rightChild.depth),
        frequency: bestCandidate.frequency,
        successRate: bestCandidate.successRate,
        xScore: bestCandidate.xScore,
      };

      this.vocabulary.set(bestCandidate.combinedSymbol, newNode);
      mergeCount += 1;
    }

    return this.vocabulary;
  }
}

/**
 * Controller 1: Offline RL Instance Sampler & Reward Evaluator (WebArena)
 * Mengubah node internal X-Tree menjadi unit latihan GRPO parsial dengan step matching & completion bonus
 */
export class XTreeOfflineRLSampler {
  private alpha: number = 0.3;
  private gamma: number = 0.5;

  constructor(alpha: number = 0.3, gamma: number = 0.5) {
    this.alpha = alpha;
    this.gamma = gamma;
  }

  public evaluateRolloutReward(
    goldActions: PrimitiveAction[],
    rolloutActions: PrimitiveAction[],
    nodeDepth: number
  ): { reward: number; isCompleted: boolean; stepMatchRatio: number } {
    const nodeLength = goldActions.length;
    let matchCount = 0;
    let perfectContinuousMatch = true;

    for (let j = 0; j < nodeLength; j++) {
      if (j >= rolloutActions.length) {
        perfectContinuousMatch = false;
        break;
      }
      const isStepMatch =
        goldActions[j].actionType === rolloutActions[j].actionType &&
        goldActions[j].targetElement === rolloutActions[j].targetElement;

      if (isStepMatch && perfectContinuousMatch) {
        matchCount += 1;
      } else {
        perfectContinuousMatch = false;
        break; // Rollout berhenti saat match_j = 0
      }
    }

    const stepMatchRatio = nodeLength > 0 ? matchCount / nodeLength : 0;
    const isCompleted = matchCount === nodeLength;
    const depthFactor = 1.0 + (this.gamma * nodeDepth);

    // Formulasi Reward Eq. 2 (arXiv:2609.32993)
    // r = (1 - alpha) * (1 / l_v) * sum(match_j) + alpha * (1 + gamma * d_v) * c_v
    const reward = ((1 - this.alpha) * stepMatchRatio) + (this.alpha * depthFactor * (isCompleted ? 1.0 : 0.0));

    return { reward, isCompleted, stepMatchRatio };
  }
}

/**
 * Controller 2: Online RLVR Adaptive Skill Bonus (ScienceWorld & WebShop)
 * Menginjeksi kontras pada sparse reward dan otomatis meluruh (annealing) saat verifier aktif
 */
export class XTreeOnlineRLVRBonus {
  private lambda0: number = 0.75;
  private wRef: number = 0.4; // Threshold saturasi verifier (ScienceWorld=0.4, WebShop=0.85)

  constructor(lambda0: number = 0.75, wRef: number = 0.4) {
    this.lambda0 = lambda0;
    this.wRef = wRef;
  }

  public calculateAdaptiveWeight(groupSuccessRate: number): number {
    const rawRatio = 1.0 - (groupSuccessRate / this.wRef);
    const clipped = Math.max(0, Math.min(1.0, rawRatio));
    return this.lambda0 * clipped;
  }

  public computeTotalReward(
    verifierOutcomeReward: number,
    matchedSkillNodes: XTreeNode[],
    groupSuccessRate: number
  ): { totalReward: number; skillBonus: number; adaptiveLambda: number } {
    const lambdaG = this.calculateAdaptiveWeight(groupSuccessRate);
    const skillBonusSum = matchedSkillNodes.reduce((acc, node) => acc + (1.0 + 0.5 * node.depth), 0);
    const skillBonus = lambdaG * skillBonusSum;
    const totalReward = verifierOutcomeReward + skillBonus;

    return { totalReward, skillBonus, adaptiveLambda: lambdaG };
  }
}

/**
 * Controller 3: On-Policy Self-Distillation (OPSD) Supervisor
 * Guru dan siswa adalah model yang sama (pi_theta), guru menerima X-Tree skills terindeks sebagai privileged context
 */
export class XTreeOPSDSupervisor {
  private beta: number = 2.0;       // Sharpness confidence gate
  private distillationWeight: number = 0.01;

  public computePerTokenGuidance(
    teacherLogProb: number,
    studentLogProb: number
  ): { deltaT: number; gateWeight: number; lossContribution: number } {
    const deltaT = teacherLogProb - studentLogProb;
    // Logistic sigmoid gate: g_t = sigma(beta * delta_t)
    const gateWeight = 1.0 / (1.0 + Math.exp(-this.beta * deltaT));
    const lossContribution = this.distillationWeight * gateWeight * deltaT;

    return { deltaT, gateWeight, lossContribution };
  }
}