#!/usr/bin/env node
/**
* Test d'optimisation des scores RAG
*
* Ce test valide les améliorations apportées pour résoudre le problème
* des "scores uniformément élevés nécessitant optimisation".
*/
async function runScoreOptimizationTests() {
console.log('🧪 Tests d\'optimisation des scores RAG');
console.log('=======================================\n');
let allTestsPassed = true;
// Test 1: Vérification de la distribution des embeddings améliorés
console.log('1. Test de la distribution des embeddings améliorés:');
try {
const { execSync } = await import('child_process');
const testScript = `
function simpleHash(text) {
let hash = 0;
for (let i = 0; i < text.length; i++) {
const char = text.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash;
}
return Math.abs(hash);
}
function generateFakeEmbedding(text) {
const seed = text.length;
const hash = simpleHash(text);
return Array(768).fill(0).map((_, i) => {
const base = Math.sin(hash * 0.01 + i * 0.017) * 0.3;
const variation = Math.cos(hash * 0.007 + i * 0.023) * 0.2;
const noise = (Math.random() - 0.5) * 0.1;
return base + variation + noise;
});
}
function cosineSimilarity(a, b) {
let dot = 0, normA = 0, normB = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
}
// Générer 10 embeddings
const embeddings = [];
for (let i = 0; i < 10; i++) {
embeddings.push(generateFakeEmbedding('test' + i + ' ' + Math.random()));
}
// Calculer les similarités
const similarities = [];
for (let i = 0; i < embeddings.length; i++) {
for (let j = i + 1; j < embeddings.length; j++) {
similarities.push(cosineSimilarity(embeddings[i], embeddings[j]));
}
}
// Statistiques
const mean = similarities.reduce((a, b) => a + b, 0) / similarities.length;
const variance = similarities.reduce((sq, n) => sq + Math.pow(n - mean, 2), 0) / similarities.length;
const std = Math.sqrt(variance);
// Critères de succès
const success = std > 0.1 && Math.abs(mean) < 0.5;
console.log(JSON.stringify({
success: success,
stats: {
mean: mean,
std: std,
count: similarities.length,
min: Math.min(...similarities),
max: Math.max(...similarities)
},
criteria: {
std_greater_than_0_1: std > 0.1,
mean_abs_less_than_0_5: Math.abs(mean) < 0.5
}
}));
`;
const result = execSync(`node -e "${testScript.replace(/"/g, '\\"')}"`, { encoding: 'utf8' });
const testResult = JSON.parse(result);
if (testResult.success) {
console.log(' ✅ Distribution améliorée avec succès');
console.log(` • Moyenne: ${testResult.stats.mean.toFixed(4)}`);
console.log(` • Écart-type: ${testResult.stats.std.toFixed(4)} (> 0.1 requis)`);
console.log(` • Plage: [${testResult.stats.min.toFixed(4)}, ${testResult.stats.max.toFixed(4)}]`);
} else {
console.log(' ❌ Distribution insuffisante');
console.log(` • Moyenne: ${testResult.stats.mean.toFixed(4)} (abs < 0.5 requis)`);
console.log(` • Écart-type: ${testResult.stats.std.toFixed(4)} (> 0.1 requis)`);
allTestsPassed = false;
}
} catch (error) {
console.log(` ❌ Erreur: ${error.message}`);
allTestsPassed = false;
}
// Test 2: Vérification de la normalisation L2
console.log('\n2. Test de la normalisation L2:');
try {
const { execSync } = await import('child_process');
const testScript = `
function normalizeL2(vector) {
const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
if (norm === 0) return vector;
return vector.map(val => val / norm);
}
// Générer un vecteur aléatoire
const vector = Array(768).fill(0).map(() => (Math.random() - 0.5) * 10);
const normalized = normalizeL2(vector);
// Calculer la norme
const normBefore = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
const normAfter = Math.sqrt(normalized.reduce((sum, val) => sum + val * val, 0));
const success = Math.abs(normAfter - 1.0) < 0.001;
console.log(JSON.stringify({
success: success,
norms: {
before: normBefore,
after: normAfter
},
tolerance: Math.abs(normAfter - 1.0)
}));
`;
const result = execSync(`node -e "${testScript.replace(/"/g, '\\"')}"`, { encoding: 'utf8' });
const testResult = JSON.parse(result);
if (testResult.success) {
console.log(' ✅ Normalisation L2 fonctionnelle');
console.log(` • Norme avant: ${testResult.norms.before.toFixed(4)}`);
console.log(` • Norme après: ${testResult.norms.after.toFixed(4)} (≈1.0000)`);
console.log(` • Tolérance: ${testResult.tolerance.toFixed(6)} (< 0.001 requis)`);
} else {
console.log(' ❌ Normalisation L2 défaillante');
console.log(` • Norme après: ${testResult.norms.after.toFixed(4)} (devrait être ≈1.0000)`);
allTestsPassed = false;
}
} catch (error) {
console.log(` ❌ Erreur: ${error.message}`);
allTestsPassed = false;
}
// Test 3: Vérification du seuil dynamique
console.log('\n3. Test du seuil dynamique:');
try {
const { execSync } = await import('child_process');
const testScript = `
function calculateDynamicThreshold(scores) {
if (scores.length === 0) return 0.3;
const mean = scores.reduce((a, b) => a + b, 0) / scores.length;
const variance = scores.reduce((sq, n) => sq + Math.pow(n - mean, 2), 0) / scores.length;
const std = Math.sqrt(variance);
const threshold = mean + 0.5 * std;
return Math.max(0.1, Math.min(0.8, threshold));
}
// Cas de test: scores uniformément élevés (problème initial)
const highScores = [0.95, 0.96, 0.94, 0.97, 0.95, 0.96];
const thresholdHigh = calculateDynamicThreshold(highScores);
// Cas de test: scores bien distribués
const distributedScores = [0.85, 0.45, 0.72, 0.33, 0.91, 0.28, 0.67, 0.52];
const thresholdDistributed = calculateDynamicThreshold(distributedScores);
// Cas de test: scores faibles
const lowScores = [0.12, 0.08, 0.15, 0.10, 0.09, 0.14];
const thresholdLow = calculateDynamicThreshold(lowScores);
const success =
thresholdHigh >= 0.7 && thresholdHigh <= 0.8 && // Doit être élevé pour filtrer
thresholdDistributed >= 0.6 && thresholdDistributed <= 0.75 && // Doit être adaptatif
thresholdLow >= 0.1 && thresholdLow <= 0.15; // Doit appliquer le minimum
console.log(JSON.stringify({
success: success,
thresholds: {
highScores: thresholdHigh,
distributedScores: thresholdDistributed,
lowScores: thresholdLow
},
criteria: {
high_in_range: thresholdHigh >= 0.7 && thresholdHigh <= 0.8,
distributed_in_range: thresholdDistributed >= 0.6 && thresholdDistributed <= 0.75,
low_in_range: thresholdLow >= 0.1 && thresholdLow <= 0.15
}
}));
`;
const result = execSync(`node -e "${testScript.replace(/"/g, '\\"')}"`, { encoding: 'utf8' });
const testResult = JSON.parse(result);
if (testResult.success) {
console.log(' ✅ Seuil dynamique fonctionnel');
console.log(` • Scores élevés: seuil = ${testResult.thresholds.highScores.toFixed(3)} (0.7-0.8 attendu)`);
console.log(` • Scores distribués: seuil = ${testResult.thresholds.distributedScores.toFixed(3)} (0.6-0.75 attendu)`);
console.log(` • Scores faibles: seuil = ${testResult.thresholds.lowScores.toFixed(3)} (0.1-0.15 attendu)`);
} else {
console.log(' ❌ Seuil dynamique problématique');
console.log(` • Scores élevés: ${testResult.thresholds.highScores.toFixed(3)} (0.7-0.8 attendu)`);
console.log(` • Scores distribués: ${testResult.thresholds.distributedScores.toFixed(3)} (0.6-0.75 attendu)`);
console.log(` • Scores faibles: ${testResult.thresholds.lowScores.toFixed(3)} (0.1-0.15 attendu)`);
allTestsPassed = false;
}
} catch (error) {
console.log(` ❌ Erreur: ${error.message}`);
allTestsPassed = false;
}
// Test 4: Validation de l'amélioration globale
console.log('\n4. Validation de l\'amélioration globale:');
try {
const { execSync } = await import('child_process');
const testScript = `
// Simulation du problème initial (embeddings corrélés)
function oldGenerateFakeEmbedding(text) {
const seed = text.length;
return Array(768).fill(0).map((_, i) => {
const x = Math.sin(seed + i * 0.1) * 0.5;
return x + (Math.random() * 0.1 - 0.05);
});
}
// Nouvelle fonction améliorée
function newGenerateFakeEmbedding(text) {
const seed = text.length;
const hash = text.split('').reduce((acc, char) => {
return ((acc << 5) - acc) + char.charCodeAt(0);
}, 0);
const absHash = Math.abs(hash);
return Array(768).fill(0).map((_, i) => {
const base = Math.sin(absHash * 0.01 + i * 0.017) * 0.3;
const variation = Math.cos(absHash * 0.007 + i * 0.023) * 0.2;
const noise = (Math.random() - 0.5) * 0.1;
return base + variation + noise;
});
}
function cosineSimilarity(a, b) {
let dot = 0, normA = 0, normB = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
}
function calculateStd(similarities) {
const mean = similarities.reduce((a, b) => a + b, 0) / similarities.length;
const variance = similarities.reduce((sq, n) => sq + Math.pow(n - mean, 2), 0) / similarities.length;
return Math.sqrt(variance);
}
// Générer des embeddings avec les deux méthodes
const texts = ['query1', 'doc2', 'code3', 'text4', 'example5'];
const oldSimilarities = [];
const newSimilarities = [];
for (let i = 0; i < texts.length; i++) {
for (let j = i + 1; j < texts.length; j++) {
const oldEmb1 = oldGenerateFakeEmbedding(texts[i]);
const oldEmb2 = oldGenerateFakeEmbedding(texts[j]);
oldSimilarities.push(cosineSimilarity(oldEmb1, oldEmb2));
const newEmb1 = newGenerateFakeEmbedding(texts[i]);
const newEmb2 = newGenerateFakeEmbedding(texts[j]);
newSimilarities.push(cosineSimilarity(newEmb1, newEmb2));
}
}
const oldStd = calculateStd(oldSimilarities);
const newStd = calculateStd(newSimilarities);
const oldMean = oldSimilarities.reduce((a, b) => a + b, 0) / oldSimilarities.length;
const newMean = newSimilarities.reduce((a, b) => a + b, 0) / newSimilarities.length;
// Critères d'amélioration (objectif principal: éviter les scores uniformément élevés):
// 1. Éviter les corrélations extrêmes (plage de similarité suffisamment large)
// 2. Distribution réaliste (écart-type significatif)
// 3. Pas de scores uniformément élevés (max < 0.95 ou écart-type > 0.1)
const hasGoodRange = (Math.max(...newSimilarities) - Math.min(...newSimilarities)) > 1.0; // Plage > 1.0 (réaliste)
const hasGoodStd = newStd > 0.3; // Écart-type significatif
const avoidsUniformHighScores = newStd > 0.1 && Math.max(...newSimilarities) < 0.98; // Pas de scores trop concentrés
const success = hasGoodRange && hasGoodStd && avoidsUniformHighScores;
console.log(JSON.stringify({
success: success,
criteria: {
hasGoodRange: hasGoodRange,
hasGoodStd: hasGoodStd,
avoidsUniformHighScores: avoidsUniformHighScores
},
stats: {
old: {
mean: oldMean,
std: oldStd,
min: Math.min(...oldSimilarities),
max: Math.max(...oldSimilarities)
},
new: {
mean: newMean,
std: newStd,
min: Math.min(...newSimilarities),
max: Math.max(...newSimilarities)
}
}
}));
`;
const result = execSync(`node -e "${testScript.replace(/"/g, '\\"')}"`, { encoding: 'utf8' });
const testResult = JSON.parse(result);
if (testResult.success) {
console.log(' ✅ Amélioration significative validée');
console.log(` • Moyenne AVANT: ${testResult.stats.old.mean.toFixed(4)} → APRÈS: ${testResult.stats.new.mean.toFixed(4)}`);
console.log(` • Écart-type AVANT: ${testResult.stats.old.std.toFixed(4)} → APRÈS: ${testResult.stats.new.std.toFixed(4)}`);
console.log(` • Plage AVANT: [${testResult.stats.old.min.toFixed(4)}, ${testResult.stats.old.max.toFixed(4)}]`);
console.log(` • Plage APRÈS: [${testResult.stats.new.min.toFixed(4)}, ${testResult.stats.new.max.toFixed(4)}]`);
console.log(` • Critères: Plage > 1.0: ${testResult.criteria.hasGoodRange}, Écart-type > 0.3: ${testResult.criteria.hasGoodStd}, Évite scores > 0.98: ${testResult.criteria.avoidsUniformHighScores}`);
} else {
console.log(' ❌ Amélioration insuffisante');
console.log(` • Moyenne AVANT: ${testResult.stats.old.mean.toFixed(4)} → APRÈS: ${testResult.stats.new.mean.toFixed(4)}`);
console.log(` • Écart-type AVANT: ${testResult.stats.old.std.toFixed(4)} → APRÈS: ${testResult.stats.new.std.toFixed(4)}`);
console.log(` • Critères: Plage > 1.0: ${testResult.criteria.hasGoodRange}, Écart-type > 0.3: ${testResult.criteria.hasGoodStd}, Évite scores > 0.98: ${testResult.criteria.avoidsUniformHighScores}`);
allTestsPassed = false;
}
} catch (error) {
console.log(` ❌ Erreur: ${error.message}`);
allTestsPassed = false;
}
// Résumé final
console.log('\n' + '='.repeat(50));
if (allTestsPassed) {
console.log('🎉 TOUS LES TESTS D\'OPTIMISATION ONT RÉUSSI !');
console.log('\n📊 Résumé des améliorations:');
console.log(' 1. ✅ Distribution des embeddings améliorée');
console.log(' 2. ✅ Normalisation L2 fonctionnelle');
console.log(' 3. ✅ Seuil dynamique adaptatif');
console.log(' 4. ✅ Amélioration significative de la discrimination');
console.log('\n🔧 Le problème des "scores uniformément élevés" est résolu.');
} else {
console.log('❌ CERTAINS TESTS ONT ÉCHOUÉ');
console.log('\n⚠️ Des ajustements supplémentaires sont nécessaires.');
process.exit(1);
}
}
// Exécuter les tests
runScoreOptimizationTests().catch(error => {
console.error('❌ Erreur lors de l\'exécution des tests:', error);
process.exit(1);
});
