import { describe, it, expect, vi, beforeEach } from 'vitest';
import { UIElementDetector, type UIElement, type DetectionConfig, type UIElementType } from '../../src/ui-element-detection.js';
import { Image } from '@nut-tree-fork/nut-js';
// Mock dependencies
vi.mock('../../src/image-utils.js', () => ({
imageToBase64: vi.fn().mockResolvedValue('base64-image-data'),
base64ToBuffer: vi.fn().mockReturnValue(Buffer.from('test-buffer'))
}));
vi.mock('../../src/ocr-utils.js', () => ({
extractTextFromImage: vi.fn().mockResolvedValue('Sample text OK Cancel'),
getTextLocations: vi.fn().mockResolvedValue([
{ text: 'OK', x: 100, y: 200, width: 50, height: 30, confidence: 95 },
{ text: 'Cancel', x: 160, y: 200, width: 60, height: 30, confidence: 92 },
{ text: 'Sample text', x: 50, y: 100, width: 120, height: 20, confidence: 88 }
])
}));
vi.mock('../../src/logger.js', () => ({
logger: {
startTimer: vi.fn(),
endTimer: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn()
}
}));
vi.mock('canvas', () => ({
createCanvas: vi.fn().mockReturnValue({
getContext: vi.fn().mockReturnValue({
drawImage: vi.fn(),
getImageData: vi.fn().mockReturnValue({
data: new Uint8ClampedArray(1000 * 800 * 4).fill(255), // White image
width: 1000,
height: 800
})
}),
constructor: class MockImage {}
}),
Canvas: vi.fn(),
CanvasRenderingContext2D: vi.fn(),
ImageData: vi.fn()
}));
describe('UIElementDetector', () => {
let detector: UIElementDetector;
let mockImage: Image;
beforeEach(() => {
detector = new UIElementDetector();
mockImage = {
width: 1000,
height: 800
} as Image;
});
describe('detectUIElements', () => {
it('should detect UI elements using multiple strategies', async () => {
const elements = await detector.detectUIElements(mockImage);
expect(elements).toBeDefined();
expect(Array.isArray(elements)).toBe(true);
expect(elements.length).toBeGreaterThan(0);
});
it('should detect button elements from text', async () => {
const elements = await detector.detectUIElements(mockImage);
const buttons = elements.filter(e => e.type === 'button');
expect(buttons.length).toBeGreaterThan(0);
const okButton = buttons.find(b => b.text === 'OK');
expect(okButton).toBeDefined();
expect(okButton?.clickable).toBe(true);
expect(okButton?.interactive).toBe(true);
expect(okButton?.x).toBe(100);
expect(okButton?.y).toBe(200);
});
it('should assign confidence scores to detected elements', async () => {
const elements = await detector.detectUIElements(mockImage);
for (const element of elements) {
expect(element.confidence).toBeGreaterThanOrEqual(0);
expect(element.confidence).toBeLessThanOrEqual(1);
}
});
it('should include detection methods for each element', async () => {
const elements = await detector.detectUIElements(mockImage);
for (const element of elements) {
expect(element.detectionMethod).toBeDefined();
expect(Array.isArray(element.detectionMethod)).toBe(true);
expect(element.detectionMethod.length).toBeGreaterThan(0);
}
});
it('should filter elements by minimum confidence', async () => {
const config: Partial<DetectionConfig> = {
minConfidence: 0.8
};
detector = new UIElementDetector(config);
const elements = await detector.detectUIElements(mockImage);
for (const element of elements) {
expect(element.confidence).toBeGreaterThanOrEqual(0.8);
}
});
});
describe('verifyElementInteractivity', () => {
it('should verify button interactivity correctly', async () => {
const buttonElement: UIElement = {
id: 'test-button',
type: 'button',
text: 'OK',
x: 100,
y: 200,
width: 50,
height: 30,
confidence: 0.9,
clickable: true,
interactive: true,
description: 'Test button',
detectionMethod: ['text_pattern'],
attributes: {},
visualFeatures: {
backgroundColor: '#007AFF',
borderRadius: 6
}
};
const verification = await detector.verifyElementInteractivity(buttonElement);
expect(verification.isInteractive).toBe(true);
expect(verification.confidence).toBeGreaterThan(0.5);
expect(verification.reasons.length).toBeGreaterThan(0);
});
it('should reject elements that are too small', async () => {
const tinyElement: UIElement = {
id: 'tiny-element',
type: 'other',
text: 'tiny',
x: 10,
y: 10,
width: 5,
height: 5,
confidence: 0.9,
clickable: false,
interactive: false,
description: 'Tiny element',
detectionMethod: ['text_pattern'],
attributes: {}
};
const verification = await detector.verifyElementInteractivity(tinyElement);
expect(verification.isInteractive).toBe(false);
expect(verification.reasons).toContain('Too small to be interactive');
});
});
describe('validateDetectedElements', () => {
it('should filter out low-confidence elements', async () => {
const elements: UIElement[] = [
{
id: 'good-element',
type: 'button',
text: 'OK',
x: 100,
y: 200,
width: 50,
height: 30,
confidence: 0.9,
clickable: true,
interactive: true,
description: 'Good button',
detectionMethod: ['text_pattern'],
attributes: {}
},
{
id: 'bad-element',
type: 'other',
text: 'x',
x: 10,
y: 10,
width: 2,
height: 2,
confidence: 0.1,
clickable: false,
interactive: false,
description: 'Bad element',
detectionMethod: ['text_pattern'],
attributes: {}
}
];
const validated = await detector.validateDetectedElements(elements);
expect(validated.length).toBe(1);
expect(validated[0].id).toBe('good-element');
expect(validated[0].attributes.isVerified).toBe(true);
});
});
describe('createElementHierarchy', () => {
it('should establish parent-child relationships', async () => {
const elements: UIElement[] = [
{
id: 'dialog',
type: 'dialog',
text: 'Confirm Action',
x: 100,
y: 100,
width: 300,
height: 200,
confidence: 0.9,
clickable: false,
interactive: false,
description: 'Dialog window',
detectionMethod: ['visual_analysis'],
attributes: {}
},
{
id: 'ok-button',
type: 'button',
text: 'OK',
x: 150,
y: 250,
width: 50,
height: 30,
confidence: 0.9,
clickable: true,
interactive: true,
description: 'OK button',
detectionMethod: ['text_pattern'],
attributes: {}
},
{
id: 'cancel-button',
type: 'button',
text: 'Cancel',
x: 220,
y: 250,
width: 60,
height: 30,
confidence: 0.9,
clickable: true,
interactive: true,
description: 'Cancel button',
detectionMethod: ['text_pattern'],
attributes: {}
}
];
const hierarchical = detector.createElementHierarchy(elements);
expect(hierarchical.length).toBe(1); // Only the dialog as root
expect(hierarchical[0].id).toBe('dialog');
expect(hierarchical[0].children).toBeDefined();
expect(hierarchical[0].children?.length).toBe(2);
const childIds = hierarchical[0].children?.map(c => c.id) || [];
expect(childIds).toContain('ok-button');
expect(childIds).toContain('cancel-button');
});
});
describe('UI element type classification', () => {
it('should classify button text correctly', async () => {
const buttonTexts = ['OK', 'Cancel', 'Save', 'Delete', 'Apply'];
for (const text of buttonTexts) {
const isButton = (detector as any).isButtonText(text);
expect(isButton).toBe(true);
}
});
it('should classify link text correctly', async () => {
const linkTexts = ['https://example.com', 'www.example.com', 'click here', 'learn more'];
for (const text of linkTexts) {
const isLink = (detector as any).isLinkText(text);
expect(isLink).toBe(true);
}
});
it('should classify text field indicators correctly', async () => {
const fieldTexts = ['Enter your name', 'Email address', 'Password', 'Search'];
for (const text of fieldTexts) {
const isField = (detector as any).isTextFieldIndicator(text);
expect(isField).toBe(true);
}
});
});
describe('Visual feature analysis', () => {
it('should detect button colors correctly', async () => {
const buttonColors = ['#007AFF', '#34C759', '#FF3B30'];
for (const color of buttonColors) {
const isButtonColor = (detector as any).isButtonColor(color);
expect(isButtonColor).toBe(true);
}
});
it('should detect text field colors correctly', async () => {
const fieldColors = ['#FFFFFF', '#F2F2F7'];
for (const color of fieldColors) {
const isFieldColor = (detector as any).isTextFieldColor(color);
expect(isFieldColor).toBe(true);
}
});
it('should calculate color similarity correctly', async () => {
const similarity1 = (detector as any).colorSimilarity('#FFFFFF', '#FFFFFF');
expect(similarity1).toBe(1);
const similarity2 = (detector as any).colorSimilarity('#FFFFFF', '#000000');
expect(similarity2).toBe(0);
const similarity3 = (detector as any).colorSimilarity('#FFFFFF', '#F0F0F0');
expect(similarity3).toBeGreaterThan(0.8);
});
});
});
