consolidate

Instructions

Implementation Reference

• src/mcp/server.ts:141-180 (registration)

  Schema and registration for the 'consolidate' tool. Defines name, description, and inputSchema (scope, min_age_days, dry_run). Listed in the TOOLS array that is exposed via ListToolsRequestSchema.

    {
      name: 'consolidate',
      description:
        'Sleep-mode compression. Clusters cold low-importance memories by (entity, layer), summarizes each cluster into a single protected learning-layer entry, deletes originals, and runs a forget-sweep. Run at session end or on demand. Set dry_run=true to preview without writing.',
      inputSchema: {
        type: 'object',
        properties: {
          scope: { type: 'string', enum: ['all', 'session'], default: 'session' },
          min_age_days: { type: 'number', description: 'Override the default 7-day minimum age for clustering (set to 0 to consolidate everything immediately, useful right after a bulk import).', default: 7 },
          dry_run: { type: 'boolean', default: false, description: 'Preview what would be compressed without modifying the DB.' },
        },
      },
    },
    {
      name: 'recall_file',
      description:
        'Get the COMPLETE edit history of a file across all sessions, with per-edit user-intent context. Returns: total edit count, daily breakdown, list of distinct user intents that drove the edits, and the linked memories. Use this when you need to understand WHY a file was modified historically — far more accurate than recall() for file-centric questions because it queries session_file_edits (every physical edit) instead of summary memories.',
      inputSchema: {
        type: 'object',
        properties: {
          path_substring: { type: 'string', description: 'Substring to match against file_path (e.g. "search-services.ts" or full absolute path)' },
          max_intents: { type: 'number', description: 'Max distinct user-intent snippets to return. Default 10.', default: 10 },
        },
        required: ['path_substring'],
      },
    },
    {
      name: 'read_smart',
      description:
        'Read a file with diff-only caching. Returns: (1) full content + chunk metadata on first read, (2) "unchanged" + cached chunk list (~50 tokens) if mtime matches, (3) "unchanged_content" if mtime changed but sha256 matches (touched but not modified), (4) changed chunks with content + unchanged chunks as metadata-only if the file was truly modified. Use INSTEAD of Read for files you have read before — saves 50%+ tokens on re-reads.',
      inputSchema: {
        type: 'object',
        properties: {
          path: { type: 'string', description: 'Absolute file path' },
          force: { type: 'boolean', description: 'If true, return full content regardless of cache state', default: false },
        },
        required: ['path'],
      },
    },
  ];

• src/mcp/server.ts:565-604 (handler)

  MCP handler for 'consolidate'. Handles dry_run preview (counts candidates without modifying DB) and delegates to the core library function runConsolidate (imported from '../lib/consolidate.js') for actual consolidation.

  function handleConsolidate(args: any): string {
    const dryRun = !!args?.dry_run;
    if (dryRun) {
      // Preview: simulate by running against a transaction we roll back
      // We don't have a clean "preview" path in lib/consolidate, so do a best-effort
      // read-only audit: count candidates using the same rules.
      const now = Math.floor(Date.now() / 1000);
      const ageCutoff = now - (args.min_age_days ?? 7) * 86400;
      const candidates = db.prepare(`
        SELECT m.entity_id, e.name as entity_name, m.layer, COUNT(*) as c
        FROM memories m
        JOIN entities e ON e.id = m.entity_id
        WHERE m.protected = 0
          AND m.importance < 0.9
          AND m.layer IN ('context', 'emotion', 'implementation')
          AND m.created_at <= ?
        GROUP BY m.entity_id, m.layer
        HAVING c >= 2
        ORDER BY c DESC
      `).all(ageCutoff) as any[];
      const totalReplaced = candidates.reduce((s, c) => s + c.c, 0);
      return JSON.stringify({
        ok: true,
        dry_run: true,
        clusters: candidates.length,
        memories_replaced_if_run: totalReplaced,
        preview: candidates.slice(0, 20).map((c) => ({
          entity: c.entity_name,
          layer: c.layer,
          count: c.c,
        })),
        hint: 'Set dry_run=false to actually consolidate.',
      });
    }
    const result = runConsolidate(db, {
      scope: args?.scope ?? 'session',
      min_age_days: typeof args?.min_age_days === 'number' ? args.min_age_days : undefined,
    });
    return JSON.stringify({ ok: true, ...result });
  }

• src/lib/consolidate.ts:43-210 (handler)

  Core implementation of the consolidate algorithm. Queries cold, unprotected memories from context/emotion/implementation layers, groups by (entity_id, layer), filters by heat score < 30, clusters of size >= 2, creates a learning-layer summary entry, deletes originals, and runs a post-consolidate forget-sweep. Returns ConsolidateResult with scanned, clustersCompressed, memoriesReplaced, memoriesDropped, learningIdsCreated.

  export function consolidate(
    db: Database.Database,
    opts: { scope?: 'all' | 'session'; min_age_days?: number } = {}
  ): ConsolidateResult {
    const now = Math.floor(Date.now() / 1000);
    const minAgeDays = opts.min_age_days ?? DEFAULT_MIN_AGE_DAYS;
    const ageCutoff = now - minAgeDays * 86400;
  
    // Fetch candidates with age threshold. 'session' scope → same query currently
    // (Day 2: session filter needs session_id plumbing; will add when sessions
    // table is actually populated by the MCP server).
    const layerPlaceholders = CLUSTER_LAYERS.map(() => '?').join(',');
    const rows = db
      .prepare(
        `
        SELECT m.id, m.entity_id, m.layer, m.content, m.importance,
               m.last_accessed_at, m.access_count, m.created_at, m.protected,
               e.name as entity_name
        FROM memories m
        JOIN entities e ON e.id = m.entity_id
        WHERE m.protected = 0
          AND m.layer IN (${layerPlaceholders})
          AND m.created_at <= ?
        ORDER BY m.entity_id, m.layer, m.created_at
        `
      )
      .all(...CLUSTER_LAYERS, ageCutoff) as Candidate[];
  
    // Filter to cold-heat memories
    const cold = rows.filter((r) => {
      const daysSince = (now - r.last_accessed_at) / 86400;
      const heat = computeHeat({
        accessesLast30d: daysSince < 30 ? r.access_count : 0,
        accessesLast90d: daysSince < 90 ? r.access_count : 0,
        daysSinceLastAccess: daysSince,
        totalAccesses: r.access_count,
        baseImportance: r.importance,
      });
      return heat.score < MAX_HEAT;
    });
  
    // Group by (entity_id, layer)
    const groups = new Map<string, Candidate[]>();
    for (const r of cold) {
      const key = `${r.entity_id}::${r.layer}`;
      const arr = groups.get(key) ?? [];
      arr.push(r);
      groups.set(key, arr);
    }
  
    const result: ConsolidateResult = {
      scanned: rows.length,
      clustersCompressed: 0,
      memoriesReplaced: 0,
      memoriesDropped: 0,
      learningIdsCreated: [],
    };
  
    const insertLearning = db.prepare(
      `INSERT INTO memories (entity_id, layer, content, importance, protected, source)
       VALUES (?, 'learning', ?, ?, 1, ?)`
    );
    const deleteMemory = db.prepare('DELETE FROM memories WHERE id = ?');
    const insertAudit = db.prepare(
      `INSERT INTO consolidations (learning_id, replaced_ids, replaced_count, entity_id, original_layer)
       VALUES (?, ?, ?, ?, ?)`
    );
    const insertEvent = db.prepare(
      'INSERT INTO events (entity_id, kind, payload) VALUES (?, ?, ?)'
    );
  
    const tx = db.transaction(() => {
      for (const [key, cluster] of groups) {
        if (cluster.length < MIN_CLUSTER_SIZE) continue;
  
        const [entityIdStr, layer] = key.split('::');
        const entityId = Number(entityIdStr);
        const entityName = cluster[0].entity_name;
  
        // Sort by importance desc, take top 3 as exemplars
        const byImp = [...cluster].sort((a, b) => b.importance - a.importance);
        const exemplars = byImp.slice(0, 3).map((c) => ({
          fragment: c.content.length > 200 ? c.content.slice(0, 200) + '…' : c.content,
          when: new Date(c.created_at * 1000).toISOString().slice(0, 10),
          importance: c.importance,
        }));
  
        const timestamps = cluster.map((c) => c.created_at);
        const earliest = new Date(Math.min(...timestamps) * 1000).toISOString().slice(0, 10);
        const latest = new Date(Math.max(...timestamps) * 1000).toISOString().slice(0, 10);
  
        const avgImp = cluster.reduce((s, c) => s + c.importance, 0) / cluster.length;
  
        const summary = {
          source: 'consolidate',
          original_layer: layer,
          count: cluster.length,
          period: { from: earliest, to: latest },
          pattern: `${cluster.length} cold observations on "${entityName}" (${layer}) consolidated during sleep`,
          exemplars,
          replaced_ids: cluster.map((c) => c.id),
        };
  
        const learningImportance = Math.max(avgImp, 0.55); // summaries are slightly more important than their avg source
        const sourceMeta = JSON.stringify({ origin: 'consolidate', replaced: cluster.length });
  
        const ins = insertLearning.run(
          entityId,
          JSON.stringify(summary, null, 2),
          learningImportance,
          sourceMeta
        );
        const learningId = Number(ins.lastInsertRowid);
        result.learningIdsCreated.push(learningId);
  
        insertAudit.run(
          learningId,
          JSON.stringify(cluster.map((c) => c.id)),
          cluster.length,
          entityId,
          layer
        );
  
        for (const c of cluster) deleteMemory.run(c.id);
        insertEvent.run(entityId, 'memory_consolidated', JSON.stringify({ learning_id: learningId, count: cluster.length, layer }));
  
        result.clustersCompressed++;
        result.memoriesReplaced += cluster.length;
      }
    });
    tx();
  
    // Post-consolidate: forget-sweep remaining non-clustered cold memories
    const remaining = db
      .prepare('SELECT id, layer, importance, access_count, last_accessed_at, protected FROM memories WHERE protected = 0')
      .all() as any[];
  
    const toDrop: number[] = [];
    for (const r of remaining) {
      const daysSince = (now - r.last_accessed_at) / 86400;
      const heat = computeHeat({
        accessesLast30d: daysSince < 30 ? r.access_count : 0,
        accessesLast90d: daysSince < 90 ? r.access_count : 0,
        daysSinceLastAccess: daysSince,
        totalAccesses: r.access_count,
        baseImportance: r.importance,
      });
      const action = decideForgetting({
        daysSinceLastAccess: daysSince,
        importance: r.importance,
        heatScore: heat.score,
        protected: r.protected === 1,
        layer: r.layer,
      });
      if (action === 'drop') toDrop.push(r.id);
    }
  
    if (toDrop.length > 0) {
      const del = db.prepare('DELETE FROM memories WHERE id = ?');
      const tx2 = db.transaction((ids: number[]) => {
        for (const id of ids) del.run(id);
      });
      tx2(toDrop);
      result.memoriesDropped = toDrop.length;
    }
  
    return result;
  }

• src/lib/consolidate.ts:17-23 (schema)

  ConsolidateResult interface defining the return shape: scanned, clustersCompressed, memoriesReplaced, memoriesDropped, learningIdsCreated.

  export interface ConsolidateResult {
    scanned: number;
    clustersCompressed: number;
    memoriesReplaced: number;
    memoriesDropped: number;
    learningIdsCreated: number[];
  }

• src/mcp/server.ts:809-821 (registration)

  Switch-case dispatch in CallToolRequestSchema handler that routes 'consolidate' tool name to handleConsolidate function.

        case 'consolidate': text = handleConsolidate(args); break;
        case 'recall_file': text = handleRecallFile(args); break;
        case 'read_smart': text = handleReadSmart(args); break;
        default: throw new Error(`Unknown tool: ${name}`);
      }
      return { content: [{ type: 'text', text }] };
    } catch (err: any) {
      return {
        content: [{ type: 'text', text: JSON.stringify({ ok: false, error: err?.message ?? String(err) }) }],
        isError: true,
      };
    }
  });