/**
* Meeting Chief Lite - Embedding Generation
* Uses OpenRouter for embeddings via OpenAI-compatible API
*/
import { getDb, insertVector, getChunks, updateEmbeddingJobStatus } from './database.js';
const EMBEDDING_MODEL = 'openai/text-embedding-3-small';
const EMBEDDING_DIMENSIONS = 1536;
const BATCH_SIZE = 100;
interface EmbeddingResponse {
data: Array<{
embedding: number[];
index: number;
}>;
model: string;
usage: {
prompt_tokens: number;
total_tokens: number;
};
}
export async function getEmbedding(text: string): Promise<Float32Array> {
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) {
throw new Error('OPENROUTER_API_KEY not set');
}
const response = await fetch('https://openrouter.ai/api/v1/embeddings', {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
'HTTP-Referer': 'https://github.com/smcdonnell7/meeting-chief-lite',
},
body: JSON.stringify({
model: EMBEDDING_MODEL,
input: text,
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Embedding API error: ${response.status} - ${error}`);
}
const data = await response.json() as EmbeddingResponse;
return new Float32Array(data.data[0].embedding);
}
export async function getEmbeddingsBatch(texts: string[]): Promise<Float32Array[]> {
const apiKey = process.env.OPENROUTER_API_KEY;
if (!apiKey) {
throw new Error('OPENROUTER_API_KEY not set');
}
const response = await fetch('https://openrouter.ai/api/v1/embeddings', {
method: 'POST',
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json',
'HTTP-Referer': 'https://github.com/smcdonnell7/meeting-chief-lite',
},
body: JSON.stringify({
model: EMBEDDING_MODEL,
input: texts,
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Embedding API error: ${response.status} - ${error}`);
}
const data = await response.json() as EmbeddingResponse;
// Sort by index to maintain order
const sorted = data.data.sort((a, b) => a.index - b.index);
return sorted.map(d => new Float32Array(d.embedding));
}
export async function generateEmbeddingsForMeeting(meetingId: string): Promise<number> {
const chunks = getChunks(meetingId);
if (chunks.length === 0) {
return 0;
}
// Check which chunks already have embeddings
const db = getDb();
const existingVectors = new Set(
(db.prepare(`
SELECT chunk_id FROM transcript_vectors
WHERE chunk_id IN (SELECT id FROM transcript_chunks WHERE meeting_id = ?)
`).all(meetingId) as { chunk_id: string }[]).map(r => r.chunk_id)
);
const chunksToEmbed = chunks.filter(c => !existingVectors.has(c.id));
if (chunksToEmbed.length === 0) {
return 0;
}
let generated = 0;
// Process in batches
for (let i = 0; i < chunksToEmbed.length; i += BATCH_SIZE) {
const batch = chunksToEmbed.slice(i, i + BATCH_SIZE);
const texts = batch.map(c => c.content);
try {
const embeddings = await getEmbeddingsBatch(texts);
for (let j = 0; j < batch.length; j++) {
insertVector(batch[j].id, embeddings[j], EMBEDDING_MODEL);
generated++;
}
} catch (error) {
console.error(`Error generating embeddings for batch ${i}:`, error);
throw error;
}
}
return generated;
}
export async function processEmbeddingJobs(): Promise<{ processed: number; failed: number }> {
const db = getDb();
const jobs = db.prepare(`
SELECT * FROM embedding_jobs
WHERE status = 'pending' AND retry_count < 3
ORDER BY created_at
LIMIT 10
`).all() as { id: string; meeting_id: string }[];
let processed = 0;
let failed = 0;
for (const job of jobs) {
try {
updateEmbeddingJobStatus(job.id, 'processing');
await generateEmbeddingsForMeeting(job.meeting_id);
updateEmbeddingJobStatus(job.id, 'completed');
processed++;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
updateEmbeddingJobStatus(job.id, 'failed', message);
failed++;
}
}
return { processed, failed };
}
export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
