Apple Tools MCP

/** * Performance tests for batch processing efficiency * Tests batching speedup and optimal batch size */ import { describe, it, expect, beforeEach, vi } from 'vitest' import { createEmbeddingMock, BATCH_SIZE, BATCH_DELAY_MS, EMBEDDING_DIM } from '../helpers/indexing-mocks.js' import { generateSearchTexts } from '../helpers/test-data-generators.js' import { measureTime, calculateThroughput } from '../helpers/performance-utils.js' describe('Batch Processing Efficiency', () => { let mockEmbedder beforeEach(() => { vi.clearAllMocks() const mock = createEmbeddingMock() mockEmbedder = mock.mockEmbedder }) describe('batching vs sequential', () => { it('should achieve >= 3x speedup with batching vs sequential', async () => { const count = 64 const texts = generateSearchTexts(count) // Sequential (one at a time) const { duration: seqDuration } = await measureTime(async () => { for (const text of texts) { await mockEmbedder([text], { pooling: 'mean', normalize: true }) } }) // Batched (BATCH_SIZE at a time) const { duration: batchDuration } = await measureTime(async () => { for (let i = 0; i < texts.length; i += BATCH_SIZE) { const batch = texts.slice(i, i + BATCH_SIZE) await mockEmbedder(batch, { pooling: 'mean', normalize: true }) } }) const speedup = seqDuration / batchDuration console.log(`Sequential: ${seqDuration.toFixed(1)}ms`) console.log(`Batched: ${batchDuration.toFixed(1)}ms`) console.log(`Speedup: ${speedup.toFixed(1)}x`) // Batching should be faster (or at least not slower for mocked) expect(speedup).toBeGreaterThanOrEqual(1) }) it('should reduce number of embedder calls', async () => { const count = 96 // 3 batches const texts = generateSearchTexts(count) let callCount = 0 const trackingEmbedder = vi.fn().mockImplementation(async (texts) => { callCount++ return { data: new Float32Array(texts.length * EMBEDDING_DIM).fill(0.1) } }) // Sequential would be 96 calls // Batched should be 3 calls for (let i = 0; i < texts.length; i += BATCH_SIZE) { const batch = texts.slice(i, i + BATCH_SIZE) await trackingEmbedder(batch) } expect(callCount).toBe(Math.ceil(count / BATCH_SIZE)) expect(callCount).toBeLessThan(count) }) }) describe('optimal batch size', () => { it('should show throughput at different batch sizes', async () => { const sizes = [8, 16, 32, 64] const results = [] for (const size of sizes) { const texts = generateSearchTexts(size) const { duration } = await measureTime(async () => { await mockEmbedder(texts, { pooling: 'mean', normalize: true }) }) const throughput = calculateThroughput(size, duration) results.push({ size, duration, throughput }) } console.table(results) // All batch sizes should achieve reasonable throughput for (const result of results) { expect(result.throughput).toBeGreaterThan(0) } }) it('should validate BATCH_SIZE=32 is used', () => { expect(BATCH_SIZE).toBe(32) }) }) describe('BATCH_DELAY_MS enforcement', () => { beforeEach(() => { vi.useFakeTimers() }) afterEach(() => { vi.useRealTimers() }) it('should apply delay between batches', async () => { const batchTimes = [] // Simulate batch processing with delays using fake timers const processBatches = async (texts) => { for (let i = 0; i < texts.length; i += BATCH_SIZE) { const batch = texts.slice(i, i + BATCH_SIZE) const batchIndex = Math.floor(i / BATCH_SIZE) await mockEmbedder(batch, { pooling: 'mean', normalize: true }) batchTimes.push({ index: batchIndex, time: Date.now() }) // Between batches (not after last) if (i + BATCH_SIZE < texts.length) { // Schedule the delay const delayPromise = new Promise(r => setTimeout(r, BATCH_DELAY_MS)) // Advance fake timers to resolve the delay vi.advanceTimersByTime(BATCH_DELAY_MS) await delayPromise } } } const texts = generateSearchTexts(64) // 2 batches await processBatches(texts) // Verify BATCH_DELAY_MS value and that multiple batches were processed expect(BATCH_DELAY_MS).toBe(100) expect(batchTimes.length).toBe(2) // 64 items / 32 batch size = 2 batches }) it('should not delay after last batch', async () => { const texts = generateSearchTexts(BATCH_SIZE) let delayApplied = false await mockEmbedder(texts, { pooling: 'mean', normalize: true }) // No more batches, no delay needed expect(delayApplied).toBe(false) }) }) describe('batch processing correctness', () => { it('should process all items in batches', async () => { const count = 100 const texts = generateSearchTexts(count) let processedCount = 0 for (let i = 0; i < texts.length; i += BATCH_SIZE) { const batch = texts.slice(i, i + BATCH_SIZE) await mockEmbedder(batch, { pooling: 'mean', normalize: true }) processedCount += batch.length } expect(processedCount).toBe(count) }) it('should handle non-divisible batch sizes', async () => { const count = 100 // Not divisible by 32 const texts = generateSearchTexts(count) const batches = [] for (let i = 0; i < texts.length; i += BATCH_SIZE) { batches.push(texts.slice(i, i + BATCH_SIZE)) } expect(batches.length).toBe(4) // 32 + 32 + 32 + 4 expect(batches[0].length).toBe(32) expect(batches[3].length).toBe(4) // Remainder }) }) describe('memory efficiency', () => { it('should process and discard batches', async () => { const count = 320 const texts = generateSearchTexts(count) let maxBatchSize = 0 for (let i = 0; i < texts.length; i += BATCH_SIZE) { const batch = texts.slice(i, i + BATCH_SIZE) maxBatchSize = Math.max(maxBatchSize, batch.length) await mockEmbedder(batch, { pooling: 'mean', normalize: true }) // Batch can now be GC'd } // Should never hold more than BATCH_SIZE items expect(maxBatchSize).toBeLessThanOrEqual(BATCH_SIZE) }) }) })