import { Image, Region } from '@nut-tree-fork/nut-js';
import { logger } from './logger.js';
import { cleanupImageProcessing } from './image-utils.js';
import { getTextLocations, extractTextFromImage, OCRTaskPriority, type TextLocation } from './ocr-utils.js';
/**
* Configuration for chunked image processing
*/
export interface ChunkedProcessingConfig {
maxChunkSize: number; // Maximum pixels per chunk (width * height)
overlapPercent: number; // Overlap between chunks (0-100)
maxConcurrency: number; // Maximum concurrent chunks being processed
memoryThresholdMB: number; // Memory threshold to pause processing
chunkingStrategy: 'grid' | 'adaptive' | 'linear';
}
/**
* Chunk information
*/
export interface ImageChunk {
id: string;
region: Region;
index: number;
totalChunks: number;
overlapsWithNext: boolean;
}
/**
* Chunked processing result
*/
export interface ChunkedResult {
text: string;
textLocations: TextLocation[];
processedChunks: number;
totalExecutionTime: number;
memoryPeak: number;
strategy: string;
}
/**
* Memory-efficient chunked image processor for large images
*/
export class ChunkedImageProcessor {
private config: ChunkedProcessingConfig;
private processingQueue: ImageChunk[] = [];
private activeProcessing = 0;
private memoryMonitor: NodeJS.Timeout | null = null;
private isPaused = false;
constructor(config: Partial<ChunkedProcessingConfig> = {}) {
this.config = {
maxChunkSize: 1920 * 1080, // 2MP default
overlapPercent: 10,
maxConcurrency: 2,
memoryThresholdMB: 500,
chunkingStrategy: 'adaptive',
...config
};
this.startMemoryMonitoring();
}
/**
* Process a large image using chunking strategy
*/
async processImage(
image: Image,
searchText?: string,
extractTextOnly = false,
priority = OCRTaskPriority.NORMAL
): Promise<ChunkedResult> {
const startTime = Date.now();
const memoryPeak = process.memoryUsage().heapUsed;
try {
logger.info('Starting chunked image processing', {
imageSize: `${image.width}x${image.height}`,
totalPixels: image.width * image.height,
searchText: searchText || 'none',
strategy: this.config.chunkingStrategy
});
// Check if chunking is needed
const totalPixels = image.width * image.height;
if (totalPixels <= this.config.maxChunkSize) {
logger.debug('Image small enough for direct processing');
return await this.processDirectly(image, searchText, extractTextOnly, priority);
}
// Generate chunks based on strategy
const chunks = this.generateChunks(image);
logger.info(`Generated ${chunks.length} chunks for processing`);
// Process chunks with concurrency control
const results = await this.processChunks(image, chunks, searchText, extractTextOnly, priority);
// Merge results from all chunks
const mergedResult = this.mergeResults(results, image);
const executionTime = Date.now() - startTime;
logger.info('Chunked image processing completed', {
processedChunks: results.length,
totalText: mergedResult.text.length,
totalLocations: mergedResult.textLocations.length,
executionTime,
memoryPeak: `${(memoryPeak / 1024 / 1024).toFixed(1) }MB`
});
return {
...mergedResult,
processedChunks: results.length,
totalExecutionTime: executionTime,
memoryPeak,
strategy: this.config.chunkingStrategy
};
} catch (error) {
logger.error('Chunked image processing failed', error as Error, {
imageSize: `${image.width}x${image.height}`,
strategy: this.config.chunkingStrategy
});
throw error;
} finally {
// Cleanup
this.processingQueue = [];
this.activeProcessing = 0;
this.isPaused = false;
// Force cleanup
setTimeout(() => cleanupImageProcessing(), 100);
}
}
/**
* Process image directly without chunking
*/
private async processDirectly(
image: Image,
searchText?: string,
extractTextOnly = false,
priority = OCRTaskPriority.NORMAL
): Promise<ChunkedResult> {
const startTime = Date.now();
try {
let text = '';
let textLocations: TextLocation[] = [];
if (extractTextOnly) {
text = await extractTextFromImage(image, undefined, priority);
} else {
textLocations = await getTextLocations(image, searchText, undefined, priority);
text = textLocations.map(loc => loc.text).join(' ');
}
return {
text,
textLocations,
processedChunks: 1,
totalExecutionTime: Date.now() - startTime,
memoryPeak: process.memoryUsage().heapUsed,
strategy: 'direct'
};
} catch (error) {
logger.error('Direct image processing failed', error as Error);
throw error;
}
}
/**
* Generate chunks based on the configured strategy
*/
private generateChunks(image: Image): ImageChunk[] {
const _chunks: ImageChunk[] = [];
const overlap = Math.floor(Math.min(image.width, image.height) * (this.config.overlapPercent / 100));
switch (this.config.chunkingStrategy) {
case 'grid':
return this.generateGridChunks(image, overlap);
case 'adaptive':
return this.generateAdaptiveChunks(image, overlap);
case 'linear':
return this.generateLinearChunks(image, overlap);
default:
return this.generateAdaptiveChunks(image, overlap);
}
}
/**
* Generate grid-based chunks
*/
private generateGridChunks(image: Image, overlap: number): ImageChunk[] {
const chunks: ImageChunk[] = [];
// Calculate optimal grid dimensions
const targetPixelsPerChunk = this.config.maxChunkSize;
const aspectRatio = image.width / image.height;
let chunkWidth = Math.floor(Math.sqrt(targetPixelsPerChunk * aspectRatio));
let chunkHeight = Math.floor(targetPixelsPerChunk / chunkWidth);
// Ensure chunks aren't too small
chunkWidth = Math.max(chunkWidth, 200);
chunkHeight = Math.max(chunkHeight, 200);
const stepsX = Math.ceil(image.width / chunkWidth);
const stepsY = Math.ceil(image.height / chunkHeight);
let index = 0;
for (let y = 0; y < stepsY; y++) {
for (let x = 0; x < stepsX; x++) {
const startX = Math.max(0, x * chunkWidth - overlap);
const startY = Math.max(0, y * chunkHeight - overlap);
const endX = Math.min(image.width, (x + 1) * chunkWidth + overlap);
const endY = Math.min(image.height, (y + 1) * chunkHeight + overlap);
chunks.push({
id: `grid_${x}_${y}`,
region: new Region(startX, startY, endX - startX, endY - startY),
index: index++,
totalChunks: stepsX * stepsY,
overlapsWithNext: x < stepsX - 1 || y < stepsY - 1
});
}
}
return chunks;
}
/**
* Generate adaptive chunks based on image characteristics
*/
private generateAdaptiveChunks(image: Image, overlap: number): ImageChunk[] {
const _chunks: ImageChunk[] = [];
// Start with a base chunk size and adapt based on image dimensions
const _totalPixels = image.width * image.height;
const _baseChunkSize = Math.sqrt(this.config.maxChunkSize);
// For very wide or tall images, use different strategies
const aspectRatio = image.width / image.height;
if (aspectRatio > 3) {
// Very wide image - use horizontal strips
return this.generateHorizontalStrips(image, overlap);
} else if (aspectRatio < 0.33) {
// Very tall image - use vertical strips
return this.generateVerticalStrips(image, overlap);
} else {
// Regular aspect ratio - use grid
return this.generateGridChunks(image, overlap);
}
}
/**
* Generate linear chunks (horizontal strips)
*/
private generateLinearChunks(image: Image, overlap: number): ImageChunk[] {
return this.generateHorizontalStrips(image, overlap);
}
/**
* Generate horizontal strips
*/
private generateHorizontalStrips(image: Image, overlap: number): ImageChunk[] {
const chunks: ImageChunk[] = [];
const stripHeight = Math.floor(this.config.maxChunkSize / image.width);
const steps = Math.ceil(image.height / stripHeight);
for (let i = 0; i < steps; i++) {
const startY = Math.max(0, i * stripHeight - overlap);
const endY = Math.min(image.height, (i + 1) * stripHeight + overlap);
chunks.push({
id: `hstrip_${i}`,
region: new Region(0, startY, image.width, endY - startY),
index: i,
totalChunks: steps,
overlapsWithNext: i < steps - 1
});
}
return chunks;
}
/**
* Generate vertical strips
*/
private generateVerticalStrips(image: Image, overlap: number): ImageChunk[] {
const chunks: ImageChunk[] = [];
const stripWidth = Math.floor(this.config.maxChunkSize / image.height);
const steps = Math.ceil(image.width / stripWidth);
for (let i = 0; i < steps; i++) {
const startX = Math.max(0, i * stripWidth - overlap);
const endX = Math.min(image.width, (i + 1) * stripWidth + overlap);
chunks.push({
id: `vstrip_${i}`,
region: new Region(startX, 0, endX - startX, image.height),
index: i,
totalChunks: steps,
overlapsWithNext: i < steps - 1
});
}
return chunks;
}
/**
* Process chunks with concurrency control and memory monitoring
*/
private async processChunks(
image: Image,
chunks: ImageChunk[],
searchText?: string,
extractTextOnly = false,
priority = OCRTaskPriority.NORMAL
): Promise<{ text: string; textLocations: TextLocation[]; chunk: ImageChunk }[]> {
const results: { text: string; textLocations: TextLocation[]; chunk: ImageChunk }[] = [];
const processingPromises = new Map<string, Promise<any>>();
for (const chunk of chunks) {
// Wait if we have too many concurrent operations
while (this.activeProcessing >= this.config.maxConcurrency || this.isPaused) {
await this.waitForAvailableSlot();
}
// Check memory before starting new chunk
await this.checkMemoryPressure();
this.activeProcessing++;
const promise = this.processChunk(image, chunk, searchText, extractTextOnly, priority)
.then(result => {
results.push(result);
this.activeProcessing--;
})
.catch(error => {
logger.error(`Chunk ${chunk.id} processing failed`, error);
this.activeProcessing--;
throw error;
});
processingPromises.set(chunk.id, promise);
}
// Wait for all chunks to complete
await Promise.all(processingPromises.values());
return results;
}
/**
* Process a single chunk
*/
private async processChunk(
image: Image,
chunk: ImageChunk,
searchText?: string,
extractTextOnly = false,
priority = OCRTaskPriority.NORMAL
): Promise<{ text: string; textLocations: TextLocation[]; chunk: ImageChunk }> {
const startTime = Date.now();
try {
logger.debug(`Processing chunk ${chunk.id}`, {
region: `${chunk.region.left},${chunk.region.top} ${chunk.region.width}x${chunk.region.height}`,
index: `${chunk.index + 1}/${chunk.totalChunks}`
});
let text = '';
let textLocations: TextLocation[] = [];
if (extractTextOnly) {
text = await extractTextFromImage(image, chunk.region, priority);
} else {
textLocations = await getTextLocations(image, searchText, chunk.region, priority);
text = textLocations.map(loc => loc.text).join(' ');
// Adjust coordinates to global image space
textLocations = textLocations.map(loc => ({
...loc,
x: loc.x + chunk.region.left,
y: loc.y + chunk.region.top
}));
}
const executionTime = Date.now() - startTime;
logger.debug(`Chunk ${chunk.id} completed`, {
textLength: text.length,
locationsFound: textLocations.length,
executionTime
});
return { text, textLocations, chunk };
} catch (error) {
logger.error(`Failed to process chunk ${chunk.id}`, error as Error);
throw error;
}
}
/**
* Merge results from all chunks
*/
private mergeResults(
results: { text: string; textLocations: TextLocation[]; chunk: ImageChunk }[],
_image: Image
): { text: string; textLocations: TextLocation[] } {
// Sort results by chunk index to maintain order
results.sort((a, b) => a.chunk.index - b.chunk.index);
// Merge text
const allText = results.map(r => r.text.trim()).filter(t => t.length > 0).join(' ');
// Merge and deduplicate text locations
const allLocations: TextLocation[] = [];
const locationMap = new Map<string, TextLocation>();
for (const result of results) {
for (const location of result.textLocations) {
// Create a key based on position and text for deduplication
const key = `${Math.round(location.x / 5) * 5}_${Math.round(location.y / 5) * 5}_${location.text.trim().toLowerCase()}`;
const existingLocation = locationMap.get(key);
if (!existingLocation || existingLocation.confidence < location.confidence) {
locationMap.set(key, location);
}
}
}
allLocations.push(...locationMap.values());
// Sort by position (top to bottom, left to right)
allLocations.sort((a, b) => {
const yDiff = a.y - b.y;
if (Math.abs(yDiff) > 10) {return yDiff;}
return a.x - b.x;
});
return {
text: allText,
textLocations: allLocations
};
}
/**
* Wait for an available processing slot
*/
private async waitForAvailableSlot(): Promise<void> {
return new Promise((resolve) => {
const checkSlot = () => {
if (this.activeProcessing < this.config.maxConcurrency && !this.isPaused) {
resolve();
} else {
setTimeout(checkSlot, 100);
}
};
checkSlot();
});
}
/**
* Check memory pressure and pause if needed
*/
private async checkMemoryPressure(): Promise<void> {
const memUsage = process.memoryUsage();
const memoryUsageMB = memUsage.heapUsed / 1024 / 1024;
if (memoryUsageMB > this.config.memoryThresholdMB) {
logger.warn('Memory pressure detected, pausing chunk processing', {
memoryUsageMB,
threshold: this.config.memoryThresholdMB
});
this.isPaused = true;
// Force garbage collection if available
if (global.gc) {
global.gc();
}
// Wait for memory to clear
await new Promise((resolve) => {
const checkMemory = () => {
const currentMem = process.memoryUsage().heapUsed / 1024 / 1024;
if (currentMem < this.config.memoryThresholdMB * 0.8) {
this.isPaused = false;
resolve(undefined);
} else {
setTimeout(checkMemory, 1000);
}
};
setTimeout(checkMemory, 2000); // Initial delay
});
logger.info('Memory pressure resolved, resuming chunk processing');
}
}
/**
* Start memory monitoring
*/
private startMemoryMonitoring(): void {
this.memoryMonitor = setInterval(() => {
const memUsage = process.memoryUsage();
const memoryUsageMB = memUsage.heapUsed / 1024 / 1024;
if (memoryUsageMB > this.config.memoryThresholdMB * 1.2) {
logger.warn('High memory usage detected during chunked processing', {
memoryUsageMB,
activeProcessing: this.activeProcessing,
queueLength: this.processingQueue.length
});
}
}, 5000);
}
/**
* Shutdown the processor
*/
shutdown(): void {
if (this.memoryMonitor) {
clearInterval(this.memoryMonitor);
this.memoryMonitor = null;
}
this.processingQueue = [];
this.activeProcessing = 0;
this.isPaused = false;
logger.info('Chunked image processor shut down');
}
/**
* Update configuration
*/
updateConfig(config: Partial<ChunkedProcessingConfig>): void {
this.config = { ...this.config, ...config };
logger.info('Chunked processor configuration updated', this.config);
}
}
// Global instance
let globalProcessor: ChunkedImageProcessor | null = null;
/**
* Get the global chunked image processor
*/
export function getChunkedImageProcessor(): ChunkedImageProcessor {
if (!globalProcessor) {
globalProcessor = new ChunkedImageProcessor();
}
return globalProcessor;
}
/**
* Shutdown the global processor
*/
export function shutdownChunkedImageProcessor(): void {
if (globalProcessor) {
globalProcessor.shutdown();
globalProcessor = null;
}
}
