import { describe, it, expect } from 'vitest';
/**
* These tests demonstrate the LLM-ergonomic approach to content patching
* They show how an LLM would use the new tools vs the old confusing approach
*/
describe('LLM Ergonomic Examples', () => {
describe('Old vs New: Simple Replace', () => {
it('shows the old confusing way', () => {
// User: "Replace TODO with DONE in the action items"
// Old way - LLM struggles with parameter confusion
const oldRequest = {
filepath: "meeting.md",
oldText: "TODO",
newText: "DONE",
targetType: "heading", // Why needed for simple replace?
target: "Action Items", // Is this the section?
heading: "Action Items" // Wait, both target AND heading?
};
// LLM is confused about which parameters to use
expect(oldRequest).toHaveProperty('targetType');
expect(oldRequest).toHaveProperty('target');
expect(oldRequest).toHaveProperty('heading');
});
it('shows the new clear way', () => {
// New way - explicit and deterministic
const newRequest = {
filepath: "meeting.md",
operation: {
type: "replace",
replace: {
pattern: "TODO",
replacement: "DONE",
options: {
scope: {
type: "section",
section_path: ["Action Items"]
}
}
}
}
};
// Clear structure that LLMs can understand
expect(newRequest.operation.type).toBe('replace');
expect(newRequest.operation.replace.pattern).toBe('TODO');
});
});
describe('Old vs New: Handling Ambiguous Headings', () => {
it('shows old way fails with duplicates', () => {
// Document has multiple "Overview" headings
// Old way - ambiguous
const oldRequest = {
filepath: "design.md",
content: "New content",
targetType: "heading",
target: "Overview", // Which Overview?
heading: "Overview", // Still ambiguous
insertAfter: true
};
// No way to disambiguate!
expect(oldRequest.target).toBe('Overview');
});
it('shows new way with deterministic paths', () => {
// New way - LLM queries structure first
const structureQuery = {
filepath: "design.md",
query: {
type: "headings",
filter: { text: "Overview" }
}
};
// Mock response shows multiple matches
const structureResponse = {
headings: [
{ text: "Overview", path: [], line: 10 },
{ text: "Overview", path: ["Implementation"], line: 45, occurrence: 1 },
{ text: "Overview", path: ["Testing"], line: 78, occurrence: 1 }
]
};
// LLM builds unambiguous operation
const newRequest = {
filepath: "design.md",
operation: {
type: "insert",
insert: {
content: "## Performance\nDetails...",
location: {
type: "heading",
heading: {
path: ["Implementation", "Overview"], // Unambiguous!
level: 2
},
position: "after"
}
}
}
};
expect(newRequest.operation.insert.location.heading.path).toEqual(["Implementation", "Overview"]);
});
});
describe('LLM Error Recovery', () => {
it('shows how LLMs handle structured errors', () => {
// Mock error response
const errorResponse = {
success: false,
error: {
code: "AMBIGUOUS_HEADING_PATH",
message: "Multiple headings match path [Implementation, Overview]",
suggestions: [
{ path: ["Implementation", "Overview"], line: 45, occurrence: 1 },
{ path: ["Implementation", "Overview"], line: 67, occurrence: 2 },
{ path: ["Implementation", "Overview"], line: 89, occurrence: 3 }
]
}
};
// LLM can programmatically handle the error
if (errorResponse.error.code === "AMBIGUOUS_HEADING_PATH") {
// LLM can:
// 1. Present options to user
// 2. Use occurrence number
// 3. Use additional context
const retryRequest = {
filepath: "design.md",
operation: {
type: "insert",
insert: {
content: "New content",
location: {
type: "heading",
heading: {
path: ["Implementation", "Overview"],
occurrence: 2 // Second occurrence
},
position: "after"
}
}
}
};
expect(retryRequest.operation.insert.location.heading.occurrence).toBe(2);
}
});
});
describe('Complex Frontmatter Updates', () => {
it('shows structured frontmatter operations', () => {
const request = {
filepath: "proposal.md",
operation: {
type: "update_frontmatter",
update_frontmatter: {
changes: {
set: {
status: "reviewed",
review_date: "2024-01-08"
},
append: {
tags: ["reviewed", "q1-2024"]
},
merge: {
metadata: {
reviewer: "ai-system",
confidence: 0.95
}
}
}
}
}
};
// Clear, structured operations
expect(request.operation.update_frontmatter.changes).toHaveProperty('set');
expect(request.operation.update_frontmatter.changes).toHaveProperty('append');
expect(request.operation.update_frontmatter.changes).toHaveProperty('merge');
});
});
describe('Why This Works for LLMs', () => {
it('demonstrates explicit over implicit', () => {
// Bad for LLMs - unclear parameter combinations
const badApi = {
heading: "maybe",
target: "maybe",
targetType: "maybe",
insertAfter: true,
insertBefore: false
// Which combination is valid?
};
// Good for LLMs - explicit structure
const goodApi = {
location: {
type: "heading",
heading: { path: ["Section", "Subsection"] },
position: "after"
}
// No ambiguity about what goes together
};
expect(goodApi.location.type).toBe('heading');
expect(goodApi.location.position).toBe('after');
});
it('demonstrates deterministic outcomes', () => {
const request = {
operation: {
type: "replace",
replace: {
pattern: "v1.0",
replacement: "v2.0",
options: {
max_replacements: 1, // Only first
case_sensitive: true, // Exact match
scope: { type: "document" }
}
}
}
};
// LLM knows exactly what will happen
expect(request.operation.replace.options.max_replacements).toBe(1);
});
});
});
