import { describe, it, expect, beforeEach, vi, beforeAll, afterAll } from "vitest"
import { OpenAICompatibleEmbedder } from "../openai-compatible"
/**
* Integration tests for OpenAI Compatible Embedder with real API calls
* These tests make actual HTTP requests to the SiliconFlow API when API key is available
* Otherwise, they use mock responses to simulate API behavior for testing purposes
*
* Environment Configuration:
* - To enable real API tests, set environment variable:
* export SILICONFLOW_API_KEY="your-actual-api-key"
* - Optionally set custom API endpoint:
* export SILICONFLOW_BASE_URL="https://api.siliconflow.cn/v1" (default)
*
* Running the tests:
* - With real API: npx vitest run --reporter=verbose src/code-index/embedders/__tests__/openai-compatible.integration.spec.ts
* - With mocks: Tests will run using mock responses automatically
*/
// Check if integration tests should run with real API or mocks
const hasApiKey = process.env['SILICONFLOW_API_KEY'] && process.env['SILICONFLOW_API_KEY'] !== 'sk-xxxxx' && process.env['SILICONFLOW_API_KEY'].length > 10
const baseUrl = process.env['SILICONFLOW_BASE_URL'] || "https://api.siliconflow.cn/v1"
const testApiKey = hasApiKey ? process.env['SILICONFLOW_API_KEY']! : "sk-test-key-for-mocks"
const testModelId = "Qwen/Qwen3-Embedding-4B"
const useRealApi = hasApiKey
// Mock data generators
const generateMockEmbedding = (text: string, dimensions: number = 1536): number[] => {
// Generate deterministic embeddings based on text content for consistent tests
const seed = text.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
const mockEmbedding: number[] = []
for (let i = 0; i < dimensions; i++) {
const x = Math.sin(seed + i * 12.9898) * 43758.5453
mockEmbedding.push(x - Math.floor(x))
}
return mockEmbedding
}
const calculateMockUsage = (texts: string[]) => {
// Rough estimation: ~4 characters per token
const totalTokens = Math.ceil(texts.join(' ').length / 4)
return {
promptTokens: totalTokens,
totalTokens: totalTokens
}
}
// Create a mock embedder class for testing when no real API is available
class MockOpenAICompatibleEmbedder {
private baseUrl: string
private apiKey: string
private modelId: string
private mockResponses: Map<string, any> = new Map()
constructor(baseUrl: string, apiKey: string, modelId?: string) {
this.baseUrl = baseUrl
this.apiKey = apiKey
this.modelId = modelId || "Qwen/Qwen3-Embedding-4B"
this.setupMockResponses()
}
get embedderInfo() {
return {
name: "openai-compatible",
}
}
private setupMockResponses() {
// Setup predefined mock responses for common test scenarios
this.mockResponses.set("invalid-api-key", {
error: {
message: "Invalid API key",
type: "invalid_request_error"
}
})
this.mockResponses.set("non-existent-model", {
error: {
message: "Model not found",
type: "invalid_request_error"
}
})
}
async createEmbeddings(texts: string[], model?: string): Promise<any> {
// Simulate API delay
await new Promise(resolve => setTimeout(resolve, 50))
// Handle error scenarios
if (this.apiKey === "invalid-api-key") {
throw new Error("Invalid API key")
}
if (model === "non-existent-model") {
throw new Error("Model not found")
}
// Handle empty texts
if (texts.length === 0) {
return {
embeddings: [],
usage: {
promptTokens: 0,
totalTokens: 0
}
}
}
// Generate mock embeddings
const embeddings = texts.map(text => generateMockEmbedding(text))
const usage = calculateMockUsage(texts)
return {
embeddings: embeddings,
usage: usage
}
}
}
beforeAll(() => {
if (!useRealApi) {
console.log("\n🔧 Using mock responses for OpenAI Compatible integration tests.")
console.log(" To enable real API tests:")
console.log(" 1. Get a SiliconFlow API key from https://siliconflow.cn")
console.log(" 2. Set environment variable: export SILICONFLOW_API_KEY=\"your-actual-api-key\"")
console.log(" 3. Optionally set custom endpoint: export SILICONFLOW_BASE_URL=\"https://api.siliconflow.cn/v1\"")
console.log(" 4. Run tests again: npx vitest run --reporter=verbose src/code-index/embedders/__tests__/openai-compatible.integration.spec.ts\n")
} else {
console.log("\n✅ Using real SiliconFlow API for integration tests")
}
})
describe("OpenAICompatibleEmbedder Integration Tests", () => {
let embedder: OpenAICompatibleEmbedder | MockOpenAICompatibleEmbedder
beforeEach(() => {
if (useRealApi) {
embedder = new OpenAICompatibleEmbedder(baseUrl, testApiKey, testModelId)
} else {
embedder = new MockOpenAICompatibleEmbedder(baseUrl, testApiKey, testModelId)
}
})
describe(useRealApi ? "Real API calls" : "Mock API responses", () => {
it("should create embeddings for a single text", async () => {
const testTexts = ["Hello, world! This is a test sentence for embedding."]
const result = await embedder.createEmbeddings(testTexts)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(1)
expect(result.embeddings[0]).toBeInstanceOf(Array)
expect(result.embeddings[0].length).toBeGreaterThan(0)
expect(result.usage).toBeDefined()
expect(result.usage.promptTokens).toBeGreaterThan(0)
expect(result.usage.totalTokens).toBeGreaterThan(0)
// Log the results for manual verification
console.log("Single text embedding result:")
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
console.log(`- First 5 embedding values: [${result.embeddings[0].slice(0, 5).join(", ")}...]`)
}, 10000) // 10 second timeout for API call
it("should create embeddings for multiple texts", async () => {
const testTexts = [
"This is the first test sentence.",
"This is the second test sentence with different content.",
"A third sentence to test batch processing."
]
const result = await embedder.createEmbeddings(testTexts)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(3)
// Check that all embeddings have the same dimensions
const firstEmbeddingLength = result.embeddings[0].length
expect(firstEmbeddingLength).toBeGreaterThan(0)
result.embeddings.forEach((embedding: number[], index: number) => {
expect(embedding).toBeInstanceOf(Array)
expect(embedding.length).toBe(firstEmbeddingLength)
expect(embedding.every((val: number) => typeof val === 'number')).toBe(true)
})
expect(result.usage).toBeDefined()
expect(result.usage.promptTokens).toBeGreaterThan(0)
expect(result.usage.totalTokens).toBeGreaterThan(0)
// Log the results for manual verification
console.log("Multiple texts embedding result:")
console.log(`- Number of embeddings: ${result.embeddings.length}`)
console.log(`- Embedding dimensions: ${firstEmbeddingLength}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
result.embeddings.forEach((embedding: number[], index: number) => {
console.log(`- Text ${index + 1} first 3 values: [${embedding.slice(0, 3).join(", ")}...]`)
})
}, 15000) // 15 second timeout for API call
it("should handle Chinese text correctly", async () => {
const testTexts = [
"你好，世界！这是一个中文测试句子。",
"人工智能是未来科技发展的重要方向。"
]
const result = await embedder.createEmbeddings(testTexts)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(2)
expect(result.embeddings[0].length).toBeGreaterThan(0)
expect(result.embeddings[1].length).toBe(result.embeddings[0].length)
expect(result.usage.promptTokens).toBeGreaterThan(0)
// Log the results for manual verification
console.log("Chinese text embedding result:")
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
}, 10000)
it("should handle mixed language text", async () => {
const testTexts = [
"Hello world, 你好世界, Bonjour le monde, こんにちは世界"
]
const result = await embedder.createEmbeddings(testTexts)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(1)
expect(result.embeddings[0].length).toBeGreaterThan(0)
expect(result.usage.promptTokens).toBeGreaterThan(0)
console.log("Mixed language embedding result:")
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
}, 10000)
it("should handle code snippets", async () => {
const testTexts = [
`function fibonacci(n) {
if (n <= 1) return n;
return fibonacci(n - 1) + fibonacci(n - 2);
}`,
`def quicksort(arr):
if len(arr) <= 1:
return arr
pivot = arr[len(arr) // 2]
left = [x for x in arr if x < pivot]
middle = [x for x in arr if x == pivot]
right = [x for x in arr if x > pivot]
return quicksort(left) + middle + quicksort(right)`
]
const result = await embedder.createEmbeddings(testTexts)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(2)
expect(result.embeddings[0].length).toBeGreaterThan(0)
expect(result.embeddings[1].length).toBe(result.embeddings[0].length)
console.log("Code snippets embedding result:")
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
}, 15000)
it("should use custom model when specified", async () => {
const testTexts = ["Test sentence for custom model."]
const customModel = "BAAI/bge-large-zh-v1.5" // Alternative model available on SiliconFlow
try {
const result = await embedder.createEmbeddings(testTexts, customModel)
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(1)
expect(result.embeddings[0].length).toBeGreaterThan(0)
console.log("Custom model embedding result:")
console.log(`- Model used: ${customModel}`)
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
} catch (error) {
// If the custom model is not available, log the error but don't fail the test
console.log(`Custom model ${customModel} may not be available:`, error)
expect(error).toBeDefined()
}
}, 10000)
it("should handle longer text content", async () => {
const longText = `
Artificial Intelligence (AI) is intelligence demonstrated by machines,
in contrast to the natural intelligence displayed by humans and animals.
Leading AI textbooks define the field as the study of "intelligent agents":
any device that perceives its environment and takes actions that maximize
its chance of achieving its goals. Colloquially, the term "artificial intelligence"
is often used to describe machines that mimic "cognitive" functions that humans
associate with the human mind, such as "learning" and "problem solving".
As machines become increasingly capable, tasks considered to require "intelligence"
are often removed from the definition of AI, a phenomenon known as the AI effect.
A quip in Tesler's Theorem says "AI is whatever hasn't been done yet."
For instance, optical character recognition is frequently excluded from things
considered to be AI, having become a routine technology.
`.trim()
const result = await embedder.createEmbeddings([longText])
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(1)
expect(result.embeddings[0].length).toBeGreaterThan(0)
expect(result.usage.promptTokens).toBeGreaterThan(50) // Should have many tokens for long text
console.log("Long text embedding result:")
console.log(`- Text length: ${longText.length} characters`)
console.log(`- Embedding dimensions: ${result.embeddings[0].length}`)
console.log(`- Usage: ${JSON.stringify(result.usage)}`)
}, 10000)
it("should handle empty text gracefully", async () => {
const result = await embedder.createEmbeddings([])
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(0)
expect(result.usage.promptTokens).toBe(0)
expect(result.usage.totalTokens).toBe(0)
})
})
describe(useRealApi ? "Error handling with real API" : "Error handling with mock responses", () => {
it("should handle invalid API key", async () => {
const invalidEmbedder = useRealApi
? new OpenAICompatibleEmbedder(baseUrl, "invalid-api-key", testModelId)
: new MockOpenAICompatibleEmbedder(baseUrl, "invalid-api-key", testModelId)
await expect(invalidEmbedder.createEmbeddings(["test"]))
.rejects.toThrow()
}, 10000)
it("should handle invalid model", async () => {
const testTexts = ["Test sentence."]
const invalidModel = "non-existent-model"
await expect(embedder.createEmbeddings(testTexts, invalidModel))
.rejects.toThrow()
}, 10000)
it("should handle invalid base URL", async () => {
const invalidEmbedder = useRealApi
? new OpenAICompatibleEmbedder("https://invalid-api-endpoint.com/v1", testApiKey, testModelId)
: new MockOpenAICompatibleEmbedder("https://invalid-api-endpoint.com/v1", testApiKey, testModelId)
// For mock version, this might not fail unless we specifically mock it
// For real API, it should fail with network error
if (useRealApi) {
await expect(invalidEmbedder.createEmbeddings(["test"]))
.rejects.toThrow()
} else {
// Mock version - this would still work since we're mocking the entire class
const result = await invalidEmbedder.createEmbeddings(["test"])
expect(result).toBeDefined()
expect(result.embeddings).toHaveLength(1)
}
}, 10000)
})
})
