// qlik-app-service.ts - Enhanced with Session Management and Selection Capabilities
import { ApiClient } from '../utils/api-client.js';
import { CacheManager } from '../utils/cache-manager.js';
import WebSocket from 'ws';
import { readFileSync, existsSync, statSync } from 'fs';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import enigma from 'enigma.js';
import https from 'https';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// Load Enigma schema
let schema: any;
try {
const schemaPath = join(__dirname, '../../node_modules/enigma.js/schemas/12.20.0.json');
schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
} catch (error) {
try {
const schemaPath = join(__dirname, '../../../node_modules/enigma.js/schemas/12.20.0.json');
schema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
} catch (fallbackError) {
console.error('Failed to load Enigma.js schema:', fallbackError);
schema = { structs: {}, enums: {} };
}
}
// ===== TYPE DEFINITIONS =====
// We'll use 'any' for Enigma objects since they have dynamic methods
// and proper typing would require the full Enigma.js type definitions
export interface SimpleObject {
id: string;
type: string;
title: string;
subtitle?: string;
isVisuallyHidden?: boolean;
dimensions?: Array<{
label: string;
fieldName: string;
dataType: string;
distinctValues?: number;
}>;
measures?: Array<{
label: string;
expression: string;
format?: string;
}>;
chartConfiguration?: any;
parentContainer?: string;
nestingLevel?: number;
lastDataUpdate?: string;
sampleData?: Array<Record<string, any>>;
dataExtractedAt?: string;
}
export interface ObjectDataSample {
objectId: string;
objectType: string;
sampleData: Array<Record<string, any>>;
totalRows: number;
columnInfo: Array<{
name: string;
type: 'dimension' | 'measure';
dataType: 'text' | 'numeric' | 'date' | 'timestamp' | 'dual';
isCalculated?: boolean;
expression?: string;
format?: any;
}>;
dataExtractedAt: string;
}
export interface ContainerInfo {
id: string;
type: string;
title: string;
isContainer: boolean;
nestedObjects: SimpleObject[];
totalNestedCount: number;
}
export interface SimpleSheet {
id: string;
title: string;
description?: string;
totalObjects: number;
visualizationObjects: SimpleObject[];
containerInfo: ContainerInfo[];
sheetDataSummary?: {
totalDataRows: number;
uniqueDimensions: string[];
uniqueMeasures: string[];
dataSourceTables: string[];
lastDataRefresh?: string;
};
}
export interface AppMetadataCompleteResult {
appInfo: {
id: string;
name: string;
owner: string;
};
sheets: SimpleSheet[];
totalVisualizationObjects: number;
extractionStats: {
totalLayoutContainers: number;
totalNestedObjects: number;
successfulExtractions: number;
failedExtractions: number;
visualizationTypeBreakdown: Record<string, number>;
containerAnalysis: {
totalContainers: number;
containersWithNested: number;
averageNestedPerContainer: number;
maxNestingDepth: number;
};
dataExtractionStats: {
objectsWithData: number;
totalDataRows: number;
hypercubesExtracted: number;
listObjectsExtracted: number;
fieldValuesExtracted: number;
dataExtractionErrors: number;
};
};
appDataSummary?: {
totalDataPoints: number;
dimensionsUsed: string[];
measuresUsed: string[];
dataSources: string[];
dataModelComplexity: 'simple' | 'moderate' | 'complex';
estimatedDataFreshness?: string;
};
restMetadata?: any;
engineMetadata?: any;
}
export interface AppMetadataOptions {
includeEngineMetadata?: boolean;
includeRestMetadata?: boolean;
includeSheets?: boolean;
includeObjects?: boolean;
includeObjectData?: boolean;
maxContainerDepth?: number;
maxDataRows?: number;
timeoutMs?: number;
extractFieldValues?: boolean;
sampleDataOnly?: boolean;
}
// ===== NEW INTERFACES FOR SESSION MANAGEMENT =====
export interface AppSession {
session: any;
global: any;
doc: any;
appId: string;
lastAccessed: Date;
currentSelections: Map<string, any>;
}
export interface SelectionRequest {
field: string;
values?: any[];
searchString?: string;
rangeSelection?: { min: any; max: any };
exclude?: boolean;
}
export interface FieldInfo {
name: string;
cardinality: number;
tags?: string[];
isNumeric?: boolean;
dataType?: string;
sampleValues?: string[];
}
// ===== MAIN SERVICE CLASS =====
export interface QlikAppServiceOptions {
platform?: 'cloud' | 'on-premise';
tenantUrl?: string;
// On-premise certificate paths
certPath?: string;
keyPath?: string;
// On-premise user context
userDirectory?: string;
userId?: string;
}
export class QlikAppService {
private apiClient: ApiClient;
private cacheManager: CacheManager;
private platform: 'cloud' | 'on-premise';
private tenantUrl: string;
// On-premise certificate configuration
private certPath?: string;
private keyPath?: string;
private userDirectory: string;
private userId: string;
private httpsAgent?: https.Agent;
// Session management
private sessions: Map<string, AppSession> = new Map();
private sessionTimeout: number = 300000; // 5 minutes
private sessionTimers: Map<string, NodeJS.Timeout> = new Map();
constructor(
apiClient: ApiClient,
cacheManager: CacheManager,
platform: 'cloud' | 'on-premise' = 'cloud',
tenantUrl: string = '',
options?: QlikAppServiceOptions
) {
this.apiClient = apiClient;
this.cacheManager = cacheManager;
this.platform = platform;
this.tenantUrl = tenantUrl;
// On-premise configuration from options or environment
this.certPath = options?.certPath || process.env.QLIK_CERT_PATH;
this.keyPath = options?.keyPath || process.env.QLIK_KEY_PATH;
this.userDirectory = options?.userDirectory || process.env.QLIK_USER_DIRECTORY || 'INTERNAL';
this.userId = options?.userId || process.env.QLIK_USER_ID || 'sa_api';
// Initialize HTTPS agent for on-premise if certificates are available
// Note: Only certPath is required - keyPath is resolved from folder
if (this.platform === 'on-premise' && this.certPath) {
this.initializeOnPremiseAgent();
}
}
/**
* Initialize HTTPS agent with certificates for on-premise
* QLIK_CERT_PATH is a folder containing client.pem, client_key.pem, root.pem
*/
private initializeOnPremiseAgent(): void {
if (!this.certPath) {
return;
}
// certPath should be a folder, not a file
let certDir = this.certPath;
let clientCertPath: string;
let clientKeyPath: string;
try {
const stat = statSync(certDir);
if (stat.isDirectory()) {
// Folder mode: certPath is directory containing certificates
clientCertPath = join(certDir, 'client.pem');
clientKeyPath = join(certDir, 'client_key.pem');
} else {
// Legacy file mode (for backwards compatibility)
clientCertPath = this.certPath;
clientKeyPath = this.keyPath || '';
}
} catch (error) {
console.error(`[QlikAppService] Certificate path not found: ${certDir}`);
return;
}
if (!existsSync(clientCertPath)) {
console.error(`[QlikAppService] client.pem not found in: ${certDir}`);
return;
}
if (!existsSync(clientKeyPath)) {
console.error(`[QlikAppService] client_key.pem not found in: ${certDir}`);
return;
}
try {
this.httpsAgent = new https.Agent({
cert: readFileSync(clientCertPath),
key: readFileSync(clientKeyPath),
rejectUnauthorized: false, // Qlik self-signed certificates
});
// Store resolved paths for WebSocket connection
this.certPath = clientCertPath;
this.keyPath = clientKeyPath;
console.error('[QlikAppService] On-premise HTTPS agent initialized with certificates');
console.error(`[QlikAppService] Cert: ${clientCertPath}`);
} catch (error) {
console.error('[QlikAppService] Failed to load certificates:', error);
}
}
/**
* Check if on-premise is properly configured
* Returns true if on-premise with certificates, false otherwise
*/
private isOnPremiseConfigured(): boolean {
return this.platform === 'on-premise' &&
!!this.certPath &&
!!this.keyPath &&
!!this.httpsAgent;
}
/**
* Check platform support and throw error if on-premise without certificates
*/
private checkPlatformSupport(operation: string): void {
if (this.platform === 'on-premise' && !this.isOnPremiseConfigured()) {
throw new Error(
`On-premise Engine API requires certificate configuration: ${operation}\n\n` +
`The Qlik Sense Enterprise (on-premise) Engine API requires:\n` +
`1. WebSocket connection on port 4747\n` +
`2. Certificate authentication (client certificate + key)\n` +
`3. X-Qlik-User header for user context\n\n` +
`Configure certificates via environment variables:\n` +
` QLIK_CERT_PATH=/path/to/client.pem\n` +
` QLIK_KEY_PATH=/path/to/client_key.pem\n` +
` QLIK_USER_DIRECTORY=INTERNAL (optional)\n` +
` QLIK_USER_ID=sa_api (optional)\n\n` +
`See: https://qlik.dev/apis/json-rpc/qix/`
);
}
}
// ===== SESSION MANAGEMENT METHODS =====
/**
* Get or create a persistent session for an app
* Supports both Cloud (Bearer token) and On-Premise (certificates)
*/
async getOrCreateSession(appId: string): Promise<AppSession> {
// Check platform support - on-premise requires certificates
this.checkPlatformSupport('getOrCreateSession');
// Return existing session if available
if (this.sessions.has(appId)) {
const session = this.sessions.get(appId)!;
session.lastAccessed = new Date();
this.resetSessionTimer(appId);
console.error(`♻️ Reusing existing session for app ${appId}`);
return session;
}
// Create new session based on platform
console.error(`🔌 Creating new session for app ${appId} (${this.platform})`);
const enigmaConfig = this.platform === 'on-premise'
? this.createOnPremiseEnigmaConfig(appId)
: this.createCloudEnigmaConfig(appId);
try {
const session = enigma.create(enigmaConfig);
const global = await session.open();
// Handle opening the doc
let doc: any;
try {
doc = await (global as any).openDoc(appId);
} catch (error: any) {
console.error('Error opening doc:', error);
throw error;
}
const appSession: AppSession = {
session,
global,
doc,
appId,
lastAccessed: new Date(),
currentSelections: new Map()
};
this.sessions.set(appId, appSession);
this.startSessionTimer(appId);
console.error(`✅ Session created for app ${appId} (${this.platform})`);
return appSession;
} catch (error) {
console.error(`❌ Failed to create session for app ${appId}:`, error);
throw error;
}
}
/**
* Create Enigma config for Qlik Cloud (Bearer token auth)
*/
private createCloudEnigmaConfig(appId: string): any {
const config = this.apiClient.getConfigInfo();
const apiKey = this.getApiKeyFromEnv();
const tenantUrl = config.tenantUrl;
const cleanTenantUrl = tenantUrl.replace('https://', '').replace('http://', '').replace('/api/v1', '');
const wsUrl = `wss://${cleanTenantUrl}/app/${appId}`;
console.error(`[Cloud] WebSocket URL: ${wsUrl}`);
return {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: { 'Authorization': `Bearer ${apiKey}` }
});
return ws as any;
}
};
}
/**
* Create Enigma config for On-Premise (certificate auth)
* Uses port 4747 and X-Qlik-User header
*/
private createOnPremiseEnigmaConfig(appId: string): any {
const config = this.apiClient.getConfigInfo();
const tenantUrl = config.tenantUrl || this.tenantUrl;
// Parse tenant URL to get hostname
let hostname = tenantUrl.replace('https://', '').replace('http://', '');
// Remove any port from the URL
hostname = hostname.split(':')[0].split('/')[0];
// On-premise Engine API uses port 4747
const wsUrl = `wss://${hostname}:4747/app/${appId}`;
console.error(`[On-Premise] WebSocket URL: ${wsUrl}`);
console.error(`[On-Premise] User: ${this.userDirectory}\\${this.userId}`);
return {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: {
'X-Qlik-User': `UserDirectory=${this.userDirectory}; UserId=${this.userId}`
},
// Use certificates for TLS
cert: this.certPath ? readFileSync(this.certPath) : undefined,
key: this.keyPath ? readFileSync(this.keyPath) : undefined,
rejectUnauthorized: false, // Qlik self-signed certificates
});
return ws as any;
}
};
}
/**
* Apply selections to an app
*/
async applySelections(
appId: string,
selections: SelectionRequest[],
clearPrevious: boolean = true
): Promise<any> {
console.error(`🎯 Applying ${selections.length} selections to app ${appId}`);
const appSession = await this.getOrCreateSession(appId);
const results = [];
try {
// Clear previous selections if requested
if (clearPrevious) {
await appSession.doc.clearAll();
appSession.currentSelections.clear();
console.error('🧹 Cleared all previous selections');
}
// Apply each selection
for (const selection of selections) {
try {
const field = await appSession.doc.getField(selection.field);
let success = false;
let method = 'unknown';
if (selection.searchString) {
// Search-based selection using selectAll with search
success = await field.selectAll(selection.searchString);
method = 'search';
console.error(`🔍 Search selection on ${selection.field}: "${selection.searchString}"`);
} else if (selection.rangeSelection) {
// Range selection
const range = {
qRangeInfo: [{
qRangeLo: selection.rangeSelection.min,
qRangeHi: selection.rangeSelection.max,
qMeasure: ''
}]
};
success = await field.selectValues(range);
method = 'range';
console.error(`📊 Range selection on ${selection.field}: ${selection.rangeSelection.min} to ${selection.rangeSelection.max}`);
} else if (selection.values) {
// Value-based selection
const qValues = selection.values.map((value: any) => {
if (typeof value === 'number') {
return { qNumber: value, qIsNumeric: true };
} else {
return { qText: String(value) };
}
});
success = await field.selectValues(qValues, false, selection.exclude || false);
method = selection.exclude ? 'exclude' : 'include';
console.error(`✅ Value selection on ${selection.field}: ${selection.values.length} values`);
}
// Store selection info
appSession.currentSelections.set(selection.field, {
...selection,
method,
appliedAt: new Date(),
success
});
results.push({
field: selection.field,
method,
success,
valuesCount: selection.values?.length
});
} catch (fieldError: any) {
console.error(`❌ Failed to apply selection on field ${selection.field}:`, fieldError);
results.push({
field: selection.field,
success: false,
error: fieldError.message || 'Field selection failed'
});
}
}
return {
success: true,
appId,
sessionId: appSession.doc.id,
results,
totalSelections: appSession.currentSelections.size
};
} catch (error) {
console.error('❌ Error applying selections:', error);
throw error;
}
}
/**
* Clear all selections
*/
async clearSelections(appId: string): Promise<any> {
console.error(`🧹 Clearing all selections for app ${appId}`);
const appSession = await this.getOrCreateSession(appId);
try {
await appSession.doc.clearAll();
appSession.currentSelections.clear();
return {
success: true,
message: 'All selections cleared'
};
} catch (error) {
console.error('❌ Error clearing selections:', error);
throw error;
}
}
/**
* Get current selections
*/
async getCurrentSelections(appId: string): Promise<any> {
console.error(`📋 Getting current selections for app ${appId}`);
const appSession = await this.getOrCreateSession(appId);
try {
// Create current selections object
const selectionsObj = await appSession.doc.createSessionObject({
qInfo: {
qType: 'CurrentSelections'
},
qSelectionObjectDef: {}
});
const layout = await selectionsObj.getLayout();
const qlikSelections = layout.qSelectionObject?.qSelections || [];
// Combine with our stored selection info
const enrichedSelections = qlikSelections.map((sel: any) => {
const storedInfo = appSession.currentSelections.get(sel.qField);
return {
field: sel.qField,
selected: sel.qSelected,
selectedCount: sel.qSelectedCount,
total: sel.qTotal,
isNumeric: sel.qIsNum,
locked: sel.qLocked,
// Add our stored metadata if available
...(storedInfo ? {
method: storedInfo.method,
appliedAt: storedInfo.appliedAt,
originalRequest: storedInfo
} : {})
};
});
return {
selections: enrichedSelections,
totalFields: enrichedSelections.length
};
} catch (error) {
console.error('❌ Error getting selections:', error);
throw error;
}
}
/**
* Get available fields in the app
*/
async getAvailableFields(appId: string): Promise<any> {
console.error(`🔍 Getting available fields for app ${appId}`);
const appSession = await this.getOrCreateSession(appId);
try {
// Create a field list object to get all fields
const fieldListObj = await appSession.doc.createSessionObject({
qInfo: {
qType: 'FieldList'
},
qFieldListDef: {
qShowSystem: false,
qShowHidden: false,
qShowSemantic: true,
qShowSrcTables: true
}
});
const layout = await fieldListObj.getLayout();
const fields = layout.qFieldList?.qItems || [];
// Get more details about each field
const fieldDetails = await Promise.all(
fields.map(async (field: any) => {
try {
const fieldObj = await appSession.doc.getField(field.qName);
const fieldInfo = await fieldObj.getCardinal();
return {
name: field.qName,
cardinality: fieldInfo.qCardinal,
tags: field.qTags || [],
isNumeric: field.qIsNumeric,
dataType: this.mapQlikDataType(field.qTags?.[0]),
// Sample some values for context
sampleValues: await this.getFieldSampleValues(appSession, field.qName, 5)
};
} catch (error) {
return {
name: field.qName,
error: 'Could not get field details'
};
}
})
);
await appSession.doc.destroySessionObject(fieldListObj.id);
return {
totalFields: fieldDetails.length,
fields: fieldDetails
};
} catch (error) {
console.error('❌ Error getting fields:', error);
throw error;
}
}
/**
* Get sample values from a field
*/
private async getFieldSampleValues(
appSession: AppSession,
fieldName: string,
limit: number = 10
): Promise<string[]> {
try {
const listObj = await appSession.doc.createSessionObject({
qInfo: { qType: 'FieldValues' },
qListObjectDef: {
qFieldDefs: [fieldName],
qInitialDataFetch: [{
qTop: 0,
qLeft: 0,
qHeight: limit,
qWidth: 1
}]
}
});
const layout = await listObj.getLayout();
const values = layout.qListObject?.qDataPages?.[0]?.qMatrix || [];
await appSession.doc.destroySessionObject(listObj.id);
return values.map((row: any) => row[0].qText);
} catch (error) {
console.error(`Could not get sample values for field ${fieldName}:`, error);
return [];
}
}
// ===== EXISTING METHODS MODIFIED TO USE SESSIONS =====
/**
* Get object data - now session-aware
* @param useSession - if true, uses persistent session with selections
*/
async getObjectData(
appId: string,
objectId: string,
options: any = {},
useSession: boolean = true
): Promise<ObjectDataSample | null> {
console.error(`[QlikAppService] Getting object data for ${objectId} (useSession: ${useSession})`);
let session: any = null;
let doc: any = null;
let shouldCloseSession = false;
try {
if (useSession) {
// Use persistent session (with selections)
const appSession = await this.getOrCreateSession(appId);
session = appSession.session;
doc = appSession.doc;
console.error(`📊 Using persistent session with ${appSession.currentSelections.size} active selections`);
} else {
// Create temporary session (original behavior)
const config = this.apiClient.getConfigInfo();
const apiKey = this.getApiKeyFromEnv();
const tenantUrl = config.tenantUrl;
const cleanTenantUrl = tenantUrl.replace('https://', '').replace('http://', '').replace('/api/v1', '');
const wsUrl = `wss://${cleanTenantUrl}/app/${appId}`;
const enigmaConfig = {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: { 'Authorization': `Bearer ${apiKey}` }
});
return ws as any;
}
};
session = enigma.create(enigmaConfig);
const global = await session.open();
// Handle opening doc
let doc: any;
try {
doc = await (global as any).openDoc(appId);
} catch (error: any) {
console.error('Error opening doc:', error);
throw error;
}
shouldCloseSession = true;
console.error(`📊 Using temporary session (no selections)`);
}
// Get the object
const originalObject = await doc.getObject(objectId);
const originalLayout = await originalObject.getLayout();
console.error(`[Object Data] Retrieved ${objectId}:`, {
type: originalLayout.qInfo?.qType,
hasHyperCube: !!originalLayout.qHyperCube,
originalMode: originalLayout.qHyperCube?.qMode || 'unknown'
});
let extractedData: any;
// Special handling for KPI objects
if (originalLayout.qInfo?.qType === 'kpi') {
console.error('[Object Data] Detected KPI object');
extractedData = await this.extractKPIDataFromObjectUltimateFix(originalLayout, originalObject);
} else if (!originalLayout.qHyperCube) {
throw new Error('Object does not contain a hypercube');
} else {
// Use direct hypercube access
extractedData = await this.extractDirectFromOriginalHypercube(originalLayout, originalObject, options);
}
// Close temporary session if needed
if (shouldCloseSession && session) {
await session.close();
}
if (!extractedData || !extractedData.data || extractedData.data.length === 0) {
console.error(`No data found for object ${objectId}`);
return null;
}
return this.convertToObjectDataSampleUltimateFix(extractedData, objectId);
} catch (error) {
if (shouldCloseSession && session) {
try {
await session.close();
} catch (closeError) {
console.error('Error closing session:', closeError);
}
}
console.error(`Failed to get object data for ${objectId}:`, error);
throw error;
}
}
// ===== EXISTING MAIN METHODS (PRESERVED) =====
async getAppMetadataComplete(
appId: string,
options: AppMetadataOptions = {}
): Promise<AppMetadataCompleteResult> {
console.error(`🔍 Getting complete metadata for app ${appId}`);
try {
const appInfo = await this.getBasicAppInfo(appId);
const result: AppMetadataCompleteResult = {
appInfo,
sheets: [],
totalVisualizationObjects: 0,
extractionStats: {
totalLayoutContainers: 0,
totalNestedObjects: 0,
successfulExtractions: 0,
failedExtractions: 0,
visualizationTypeBreakdown: {},
containerAnalysis: {
totalContainers: 0,
containersWithNested: 0,
averageNestedPerContainer: 0,
maxNestingDepth: 0
},
dataExtractionStats: {
objectsWithData: 0,
totalDataRows: 0,
hypercubesExtracted: 0,
listObjectsExtracted: 0,
fieldValuesExtracted: 0,
dataExtractionErrors: 0
}
}
};
if (options.includeRestMetadata !== false) {
try {
result.restMetadata = await this.apiClient.getAppMetadata(appId);
} catch (error) {
console.warn('Failed to get REST metadata:', error);
}
}
if (options.includeSheets !== false && options.includeObjects !== false) {
try {
const sheetsData = await this.getAllObjectsWithUltimateFixFromEngineAPI(appId, options);
result.sheets = sheetsData;
result.totalVisualizationObjects = sheetsData.reduce(
(sum, sheet) => sum + sheet.totalObjects, 0
);
result.extractionStats = this.buildExtractionStatsWithData(sheetsData);
if (options.includeObjectData) {
result.appDataSummary = this.buildAppDataSummary(sheetsData);
}
} catch (error) {
console.error('Failed to get Engine objects:', error);
throw error;
}
}
return result;
} catch (error) {
console.error(`Failed to get complete metadata for app ${appId}:`, error);
throw error;
}
}
async analyzeDataModel(appId: string): Promise<any> {
console.error(`[QlikAppService] Analyzing data model for app ${appId}`);
let session: any = null;
try {
// Use temporary session for data model analysis
const config = this.apiClient.getConfigInfo();
const apiKey = this.getApiKeyFromEnv();
const tenantUrl = config.tenantUrl;
const cleanTenantUrl = tenantUrl.replace('https://', '').replace('http://', '').replace('/api/v1', '');
const wsUrl = `wss://${cleanTenantUrl}/app/${appId}`;
const enigmaConfig = {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: { 'Authorization': `Bearer ${apiKey}` }
});
return ws as any;
}
};
session = enigma.create(enigmaConfig);
const global = await session.open();
let doc: any;
try {
doc = await (global as any).openDoc(appId);
} catch (error: any) {
console.error('Error opening doc:', error);
throw error;
}
console.error(`✅ Connected to app ${appId} for data model analysis`);
// Get table structure
const tablesAndKeys = await doc.getTablesAndKeys({
qcx: 1000,
qcy: 1000
});
await session.close();
const analysis = {
tables: tablesAndKeys.qtr || [],
keys: tablesAndKeys.qk || [],
issues: {
syntheticKeys: [],
circularReferences: [],
islandTables: []
}
};
// Analyze for synthetic keys
analysis.issues.syntheticKeys = analysis.keys.filter(
(key: any) => key.qKeyFields && key.qKeyFields.length > 1
);
// Analyze for island tables
const connectedTables = new Set();
analysis.keys.forEach((key: any) => {
if (key.qTables) {
key.qTables.forEach((table: string) => connectedTables.add(table));
}
});
analysis.issues.islandTables = analysis.tables.filter(
(table: any) => !connectedTables.has(table.qName)
);
return analysis;
} catch (error) {
if (session) {
try {
await session.close();
} catch (closeError) {
console.error('Error closing session:', closeError);
}
}
console.error(`Failed to analyze data model for app ${appId}:`, error);
throw error;
}
}
async fetchHypercubeData(
appId: string,
hypercubeDef: any,
maxRows: number = 1000
): Promise<any> {
console.error(`[QlikAppService] Fetching hypercube data for app ${appId}`);
let session: any = null;
try {
// Create temporary session for hypercube data
const config = this.apiClient.getConfigInfo();
const apiKey = this.getApiKeyFromEnv();
const tenantUrl = config.tenantUrl;
const cleanTenantUrl = tenantUrl.replace('https://', '').replace('http://', '').replace('/api/v1', '');
const wsUrl = `wss://${cleanTenantUrl}/app/${appId}`;
const enigmaConfig = {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: { 'Authorization': `Bearer ${apiKey}` }
});
return ws as any;
}
};
session = enigma.create(enigmaConfig);
const global = await session.open();
let doc: any;
try {
doc = await (global as any).openDoc(appId);
} catch (error: any) {
console.error('Error opening doc:', error);
throw error;
}
console.error(`✅ Connected to app ${appId}`);
const tempObject = await doc.createSessionObject({
qInfo: {
qType: 'temp-hypercube'
},
qHyperCubeDef: hypercubeDef
});
const layout = await tempObject.getLayout();
const hyperCube = layout.qHyperCube;
if (!hyperCube) {
throw new Error('No hypercube found in layout');
}
const totalRows = hyperCube.qSize.qcy;
const totalCols = hyperCube.qSize.qcx;
const rowsToFetch = Math.min(totalRows, maxRows);
console.error(`📊 Hypercube size: ${totalRows} rows x ${totalCols} cols`);
console.error(`📊 Fetching ${rowsToFetch} rows...`);
const dataPages = await tempObject.getHyperCubeData('/qHyperCubeDef', [{
qTop: 0,
qLeft: 0,
qWidth: totalCols,
qHeight: rowsToFetch
}]);
if (!dataPages || dataPages.length === 0 || !dataPages[0].qMatrix) {
throw new Error('No data returned from hypercube');
}
console.error(`✅ Retrieved ${dataPages[0].qMatrix.length} rows of data`);
const formattedData = this.formatHypercubeData(dataPages[0], hyperCube);
await doc.destroySessionObject(tempObject.id);
await session.close();
return formattedData;
} catch (error) {
console.error('❌ Error fetching hypercube data:', error);
if (session) {
try {
await session.close();
} catch (e) {
console.error('Error closing session:', e);
}
}
throw error;
}
}
// ===== SESSION MANAGEMENT HELPERS =====
/**
* Session timer management
*/
private startSessionTimer(appId: string): void {
this.resetSessionTimer(appId);
}
private resetSessionTimer(appId: string): void {
// Clear existing timer
if (this.sessionTimers.has(appId)) {
clearTimeout(this.sessionTimers.get(appId)!);
}
// Set new timer
const timer = setTimeout(() => {
console.error(`⏰ Session timeout for app ${appId}`);
this.closeSession(appId);
}, this.sessionTimeout);
this.sessionTimers.set(appId, timer);
}
/**
* Close a specific session
*/
async closeSession(appId: string): Promise<void> {
const appSession = this.sessions.get(appId);
if (appSession) {
try {
await appSession.session.close();
console.error(`🔌 Closed session for app ${appId}`);
} catch (error) {
console.error(`Error closing session for app ${appId}:`, error);
}
this.sessions.delete(appId);
if (this.sessionTimers.has(appId)) {
clearTimeout(this.sessionTimers.get(appId)!);
this.sessionTimers.delete(appId);
}
}
}
/**
* Close all sessions (for cleanup)
*/
async closeAllSessions(): Promise<void> {
console.error('🔌 Closing all sessions...');
const closePromises = Array.from(this.sessions.keys()).map(appId =>
this.closeSession(appId)
);
await Promise.all(closePromises);
}
// Override the existing closeConnections method
async closeConnections(): Promise<void> {
await this.closeAllSessions();
}
// ===== MCP HANDLERS =====
async handleGetAppMetadataComplete(args: any): Promise<{ content: { type: string; text: string }[] }> {
if (!args?.appId) {
return {
content: [{
type: 'text',
text: JSON.stringify({
error: 'App ID is required',
appId: null,
result: null
}, null, 2)
}]
};
}
try {
const result = await this.getAppMetadataComplete(args.appId, {
includeRestMetadata: args.includeRestMetadata !== false,
includeEngineMetadata: args.includeEngineMetadata !== false,
includeSheets: args.includeSheets !== false,
includeObjects: args.includeObjects !== false,
includeObjectData: args.includeObjectData === true,
maxDataRows: args.maxDataRows || 100,
extractFieldValues: args.extractFieldValues === true,
sampleDataOnly: args.sampleDataOnly !== false
});
return {
content: [{
type: 'text',
text: JSON.stringify({
appId: args.appId,
extractedSuccessfully: true,
result: {
appInfo: result.appInfo,
totalSheets: result.sheets.length,
totalVisualizationObjects: result.totalVisualizationObjects,
extractionStats: result.extractionStats,
sheets: result.sheets.map(sheet => ({
id: sheet.id,
title: sheet.title,
totalObjects: sheet.totalObjects,
visualizationObjects: sheet.visualizationObjects.map(obj => ({
id: obj.id,
type: obj.type,
title: obj.title,
subtitle: obj.subtitle,
dimensions: obj.dimensions?.map(d => ({
label: d.label,
fieldName: d.fieldName,
dataType: d.dataType,
distinctValues: d.distinctValues
})),
measures: obj.measures?.map(m => ({
label: m.label,
expression: m.expression,
format: m.format
})),
chartConfiguration: obj.chartConfiguration,
parentContainer: obj.parentContainer,
nestingLevel: obj.nestingLevel,
lastDataUpdate: obj.lastDataUpdate
})),
containerInfo: sheet.containerInfo.map(container => ({
id: container.id,
type: container.type,
title: container.title,
isContainer: container.isContainer,
totalNestedCount: container.totalNestedCount,
nestedObjectTypes: container.nestedObjects.map(obj => obj.type)
}))
})),
restMetadata: result.restMetadata,
engineMetadata: result.engineMetadata
},
generatedAt: new Date().toISOString()
}, null, 2)
}]
};
} catch (error) {
return {
content: [{
type: 'text',
text: JSON.stringify({
error: 'Metadata extraction failed',
message: error instanceof Error ? error.message : String(error),
appId: args.appId,
suggestion: 'Check Engine API permissions and app structure'
}, null, 2)
}]
};
}
}
async handleGetObjectData(args: any): Promise<{ content: { type: string; text: string }[] }> {
if (!args?.appId || !args?.objectId) {
return {
content: [{
type: 'text',
text: JSON.stringify({
error: 'App ID and Object ID are required',
appId: args?.appId || null,
objectId: args?.objectId || null,
result: null
}, null, 2)
}]
};
}
try {
// Use session if specified (default: true for filtered data)
const useSession = args.useSession !== false;
const objectData = await this.getObjectData(args.appId, args.objectId, {
maxRows: args.maxRows || 100,
includeFieldValues: args.includeFieldValues !== false,
sampleOnly: args.sampleOnly !== false
}, useSession);
if (!objectData) {
return {
content: [{
type: 'text',
text: JSON.stringify({
appId: args.appId,
objectId: args.objectId,
result: null,
message: 'No data found for this object or object does not exist',
suggestion: 'Verify the object ID and ensure the object contains data'
}, null, 2)
}]
};
}
// Get current selections if using session
let currentSelections = null;
if (useSession) {
try {
currentSelections = await this.getCurrentSelections(args.appId);
} catch (e) {
console.error('Could not get current selections:', e);
}
}
return {
content: [{
type: 'text',
text: JSON.stringify({
appId: args.appId,
objectId: args.objectId,
useSession,
currentSelections,
objectData: {
objectType: objectData.objectType,
totalRows: objectData.totalRows,
sampleRows: objectData.sampleData.length,
columnInfo: objectData.columnInfo,
sampleData: objectData.sampleData.slice(0, 10),
dataExtractedAt: objectData.dataExtractedAt
},
extractionOptions: {
maxRows: args.maxRows || 100,
includeFieldValues: args.includeFieldValues !== false,
sampleOnly: args.sampleOnly !== false
},
generatedAt: new Date().toISOString()
}, null, 2)
}]
};
} catch (error) {
return {
content: [{
type: 'text',
text: JSON.stringify({
error: 'Object data extraction failed',
message: error instanceof Error ? error.message : String(error),
appId: args.appId,
objectId: args.objectId,
suggestion: 'Check Engine API permissions and verify object exists'
}, null, 2)
}]
};
}
}
// ===== EXISTING PRIVATE METHODS (PRESERVED) =====
private async getBasicAppInfo(appId: string): Promise<{ id: string; name: string; owner: string; }> {
try {
const basicApp = await this.apiClient.getApp(appId);
const ownerId = basicApp.owner || basicApp.attributes?.owner;
const resolvedUsers = await this.apiClient.resolveOwnersToUsers([ownerId]);
const resolvedUser = resolvedUsers.get(ownerId);
return {
id: appId,
name: basicApp.name || basicApp.attributes?.name || 'Unknown App',
owner: resolvedUser?.displayName || ownerId
};
} catch (error) {
console.error(`Failed to get basic app info for ${appId}:`, error);
throw error;
}
}
private getApiKeyFromEnv(): string {
const apiKey = process.env.QLIK_API_KEY;
if (!apiKey) {
throw new Error('QLIK_API_KEY environment variable is required');
}
return apiKey;
}
private async getAllObjectsWithUltimateFixFromEngineAPI(
appId: string,
options: AppMetadataOptions
): Promise<SimpleSheet[]> {
console.error(`🔌 Connecting to Engine API for app ${appId}`);
const config = this.apiClient.getConfigInfo();
const apiKey = this.getApiKeyFromEnv();
const tenantUrl = config.tenantUrl;
const cleanTenantUrl = tenantUrl.replace('https://', '').replace('http://', '').replace('/api/v1', '');
const wsUrl = `wss://${cleanTenantUrl}/app/${appId}`;
const enigmaConfig = {
schema,
url: wsUrl,
createSocket: (url: string) => {
const ws = new WebSocket(url, {
headers: { 'Authorization': `Bearer ${apiKey}` }
});
return ws as any;
}
};
let session: any = null;
try {
session = enigma.create(enigmaConfig);
const global = await session.open();
let doc: any;
try {
doc = await (global as any).openDoc(appId);
} catch (error: any) {
console.error('Error opening doc:', error);
throw error;
}
console.error(`✅ Connected to Engine API`);
const sheets = await this.getSheetsWithUltimateFix(doc, options);
await session.close();
console.error(`🔌 Engine API connection closed`);
return sheets;
} catch (error) {
if (session) {
try {
await session.close();
} catch (closeError) {
console.error('Error closing session:', closeError);
}
}
throw error;
}
}
private async getSheetsWithUltimateFix(doc: any, options: AppMetadataOptions): Promise<SimpleSheet[]> {
console.error(`📄 Getting sheets`);
try {
const sessionObjectDef = {
qInfo: {
qType: 'SheetsContainer',
qId: ''
},
qAppObjectListDef: {
qType: 'sheet',
qData: {
title: '/qMeta/title',
description: '/qMeta/description',
rank: '/rank',
thumbnail: '/qMeta/dynamicColor'
}
}
};
const sessionObject = await doc.createSessionObject(sessionObjectDef);
const layout = await sessionObject.getLayout();
const sheetList = layout.qAppObjectList.qItems || [];
console.error(`Found ${sheetList.length} sheets`);
const sheets: SimpleSheet[] = [];
for (const sheetItem of sheetList) {
try {
const sheet = await this.extractSheetDataWithObjectsUltimateFix(
doc,
sheetItem,
options
);
sheets.push(sheet);
} catch (error) {
console.error(`Failed to extract sheet ${sheetItem.qInfo.qId}:`, error);
}
}
await doc.destroySessionObject(sessionObject.id);
return sheets;
} catch (error) {
console.error('Failed to get sheets:', error);
throw error;
}
}
private async extractSheetDataWithObjectsUltimateFix(
doc: any,
sheetItem: any,
options: AppMetadataOptions
): Promise<SimpleSheet> {
const sheetId = sheetItem.qInfo.qId;
console.error(`📊 Extracting objects from sheet: ${sheetId}`);
try {
const sheetObject = await doc.getObject(sheetId);
const sheetLayout = await sheetObject.getLayout();
const sheetData: SimpleSheet = {
id: sheetId,
title: sheetLayout.qMeta?.title || sheetItem.qData?.title || 'Untitled Sheet',
description: sheetLayout.qMeta?.description || sheetItem.qData?.description || '',
totalObjects: 0,
visualizationObjects: [],
containerInfo: []
};
if (sheetLayout.cells) {
const { extractedObjects, containerInfo } = await this.extractObjectsFromCellsUltimateFix(
doc,
sheetLayout.cells,
options
);
sheetData.visualizationObjects = extractedObjects;
sheetData.containerInfo = containerInfo;
sheetData.totalObjects = extractedObjects.length;
}
// Add summary if data was extracted
if (options.includeObjectData) {
const allDimensions = new Set<string>();
const allMeasures = new Set<string>();
let totalDataRows = 0;
sheetData.visualizationObjects.forEach(obj => {
obj.dimensions?.forEach(d => allDimensions.add(d.label));
obj.measures?.forEach(m => allMeasures.add(m.label));
if (obj.sampleData) {
totalDataRows += obj.sampleData.length;
}
});
sheetData.sheetDataSummary = {
totalDataRows,
uniqueDimensions: Array.from(allDimensions),
uniqueMeasures: Array.from(allMeasures),
dataSourceTables: [],
lastDataRefresh: new Date().toISOString()
};
}
return sheetData;
} catch (error) {
console.error(`Failed to extract sheet data for ${sheetId}:`, error);
throw error;
}
}
private async extractObjectsFromCellsUltimateFix(
doc: any,
cells: any[],
options: AppMetadataOptions
): Promise<{ extractedObjects: SimpleObject[], containerInfo: ContainerInfo[] }> {
const extractedObjects: SimpleObject[] = [];
const containerInfo: ContainerInfo[] = [];
const processedIds = new Set<string>();
for (const cell of cells) {
try {
if (!cell.name || !cell.type) continue;
if (processedIds.has(cell.name)) continue;
processedIds.add(cell.name);
if (cell.type === 'container' || cell.type === 'layoutcontainer') {
const container = await this.extractContainerWithNestedObjectsUltimateFix(
doc,
cell,
options,
0
);
if (container) {
containerInfo.push(container);
extractedObjects.push(...container.nestedObjects);
}
} else {
const simpleObj = await this.extractSingleObjectUltimateFix(
doc,
cell.name,
cell.type,
options
);
if (simpleObj) {
extractedObjects.push(simpleObj);
}
}
} catch (error) {
console.error(`Failed to extract object ${cell.name}:`, error);
}
}
return { extractedObjects, containerInfo };
}
private async extractSingleObjectUltimateFix(
doc: any,
objectId: string,
objectType: string,
options: AppMetadataOptions,
parentContainer?: string,
nestingLevel: number = 0
): Promise<SimpleObject | null> {
try {
const obj = await doc.getObject(objectId);
const layout = await obj.getLayout();
const simpleObject: SimpleObject = {
id: objectId,
type: objectType || layout.qInfo?.qType || 'unknown',
title: layout.title || layout.qMeta?.title || '',
subtitle: layout.subtitle || '',
parentContainer,
nestingLevel
};
// Extract hypercube information
if (layout.qHyperCube && options.includeObjectData) {
const hypercubeInfo = await this.extractHypercubeInfoUltimateFix(
layout.qHyperCube,
obj,
options
);
simpleObject.dimensions = hypercubeInfo.dimensions;
simpleObject.measures = hypercubeInfo.measures;
if (hypercubeInfo.sampleData && hypercubeInfo.sampleData.length > 0) {
simpleObject.sampleData = hypercubeInfo.sampleData;
simpleObject.dataExtractedAt = new Date().toISOString();
}
}
// Extract list object information (for filter panes)
if (layout.qListObject && options.includeObjectData) {
const listInfo = await this.extractListObjectInfo(layout.qListObject, obj, options);
simpleObject.dimensions = [{
label: layout.title || 'List',
fieldName: layout.qListObject.qDimensionInfo?.qGroupFieldDefs?.[0] || '',
dataType: 'text',
distinctValues: listInfo.cardinality
}];
if (listInfo.sampleValues) {
simpleObject.sampleData = listInfo.sampleValues.map((v: any) => ({ value: v }));
}
}
return simpleObject;
} catch (error) {
console.error(`Failed to extract object ${objectId}:`, error);
return null;
}
}
private async extractContainerWithNestedObjectsUltimateFix(
doc: any,
containerCell: any,
options: AppMetadataOptions,
currentDepth: number = 0
): Promise<ContainerInfo | null> {
if (currentDepth >= (options.maxContainerDepth || 3)) {
console.warn(`Max container depth reached for ${containerCell.name}`);
return null;
}
try {
const containerObj = await doc.getObject(containerCell.name);
const containerLayout = await containerObj.getLayout();
const container: ContainerInfo = {
id: containerCell.name,
type: containerCell.type,
title: containerLayout.title || containerLayout.qMeta?.title || 'Container',
isContainer: true,
nestedObjects: [],
totalNestedCount: 0
};
// Process nested objects
if (containerLayout.cells && Array.isArray(containerLayout.cells)) {
for (const nestedCell of containerLayout.cells) {
if (nestedCell.name && nestedCell.type) {
const nestedObj = await this.extractSingleObjectUltimateFix(
doc,
nestedCell.name,
nestedCell.type,
options,
containerCell.name,
currentDepth + 1
);
if (nestedObj) {
container.nestedObjects.push(nestedObj);
}
}
}
}
container.totalNestedCount = container.nestedObjects.length;
return container;
} catch (error) {
console.error(`Failed to extract container ${containerCell.name}:`, error);
return null;
}
}
private async extractHypercubeInfoUltimateFix(
hypercube: any,
objHandle: any,
options: AppMetadataOptions
): Promise<any> {
const dimensions: any[] = [];
const measures: any[] = [];
let sampleData: any[] = [];
// Extract dimensions
if (hypercube.qDimensionInfo) {
hypercube.qDimensionInfo.forEach((dimInfo: any, index: number) => {
dimensions.push({
label: dimInfo.qFallbackTitle || `Dimension ${index + 1}`,
fieldName: dimInfo.qGroupFieldDefs?.[0] || '',
dataType: this.mapQlikDataType(dimInfo.qTags?.[0]),
distinctValues: dimInfo.qCardinal
});
});
}
// Extract measures with REAL definitions
if (hypercube.qMeasureInfo) {
hypercube.qMeasureInfo.forEach((measureInfo: any, index: number) => {
console.error(`[Measure ${index}] REAL expression:`, measureInfo.qDef?.qDef || measureInfo.qDef);
measures.push({
label: measureInfo.qFallbackTitle || `Measure ${index + 1}`,
expression: measureInfo.qDef?.qDef || measureInfo.qDef || 'Unknown',
format: measureInfo.qNumFormat ? {
type: this.mapQlikNumberFormat(measureInfo.qNumFormat.qType),
pattern: measureInfo.qNumFormat.qFmt,
decimals: measureInfo.qNumFormat.qDec
} : undefined,
min: measureInfo.qMin,
max: measureInfo.qMax
});
});
}
const totalRows = hypercube.qSize?.qcy || 0;
// Extract sample data if requested
if (options.sampleDataOnly && totalRows > 0) {
try {
const maxSampleRows = Math.min(options.maxDataRows || 100, totalRows);
// Try to use existing data first
let dataMatrix: any[] = [];
if (hypercube.qDataPages && hypercube.qDataPages.length > 0 && hypercube.qDataPages[0].qMatrix) {
dataMatrix = hypercube.qDataPages[0].qMatrix.slice(0, maxSampleRows);
} else {
// Fetch fresh data if needed
const dataPages = await objHandle.getHyperCubeData('/qHyperCubeDef', [{
qTop: 0,
qLeft: 0,
qWidth: -1,
qHeight: maxSampleRows
}]);
if (dataPages && dataPages[0]?.qMatrix) {
dataMatrix = dataPages[0].qMatrix;
}
}
if (dataMatrix.length > 0) {
sampleData = dataMatrix.map((row: any) => {
const rowData: Record<string, any> = {};
// Process dimensions
dimensions.forEach((dim, dimIndex) => {
if (row[dimIndex]) {
rowData[dim.label] = row[dimIndex].qText || '';
}
});
// Process measures
measures.forEach((measure, measureIndex) => {
const cellIndex = dimensions.length + measureIndex;
if (row[cellIndex]) {
const cell = row[cellIndex];
if (typeof cell.qNum === 'number' && !isNaN(cell.qNum)) {
rowData[measure.label] = cell.qNum;
} else {
rowData[measure.label] = cell.qText || 0;
}
}
});
return rowData;
});
}
} catch (error) {
console.error('Failed to extract sample data:', error);
}
}
return {
dimensions,
measures,
sampleData,
totalRows
};
}
private async extractListObjectInfo(listObject: any, objHandle: any, options: AppMetadataOptions): Promise<any> {
const result = {
cardinality: listObject.qDimensionInfo?.qCardinal || 0,
sampleValues: [] as string[]
};
if (options.sampleDataOnly && listObject.qDataPages && listObject.qDataPages[0]) {
const dataPage = listObject.qDataPages[0];
result.sampleValues = dataPage.qMatrix
.slice(0, options.maxDataRows || 10)
.map((row: any) => row[0]?.qText || '');
}
return result;
}
// Data extraction methods for getObjectData
private async extractDirectFromOriginalHypercube(
originalLayout: any,
originalObject: any,
options: any = {}
): Promise<any> {
console.error('[Direct Hypercube] Extracting data directly from original hypercube');
const originalHyperCube = originalLayout.qHyperCube;
const maxRows = Math.min(options.maxRows || 1000, 10000);
// Get the actual data
const data = await this.extractDataFromOriginalHypercube(originalHyperCube, originalObject);
const extractedData = {
objectType: originalLayout.qInfo?.qType || 'unknown',
dimensions: originalHyperCube.qDimensionInfo?.map((dim: any, index: number) => ({
label: dim.qFallbackTitle,
fieldName: dim.qGroupFieldDefs?.[0] || dim.qFieldDefs?.[0],
position: index,
type: 'dimension'
})) || [],
measures: originalHyperCube.qMeasureInfo?.map((measure: any, index: number) => ({
label: measure.qFallbackTitle,
expression: measure.qDef?.qDef || 'Unknown',
position: index + (originalHyperCube.qDimensionInfo?.length || 0),
type: 'measure'
})) || [],
data: data,
totalRows: data.length,
extractionMetadata: {
timestamp: new Date().toISOString(),
objectType: originalLayout.qInfo?.qType,
originalMode: originalHyperCube.qMode,
extractionMode: 'direct-hypercube-access'
}
};
return extractedData;
}
private async extractDataFromOriginalHypercube(
hyperCube: any,
object: any
): Promise<any[]> {
console.error('[Extract Data] Extracting data from original hypercube');
const needsDataFetch = !hyperCube.qDataPages ||
hyperCube.qDataPages.length === 0 ||
!hyperCube.qDataPages[0]?.qMatrix ||
hyperCube.qDataPages[0].qMatrix.length === 0;
let dataMatrix: any[] = [];
if (needsDataFetch && hyperCube.qSize && hyperCube.qSize.qcy > 0) {
console.error(`[Extract Data] Fetching ${hyperCube.qSize.qcy} rows...`);
try {
const dataPages = await object.getHyperCubeData('/qHyperCubeDef', [{
qTop: 0,
qLeft: 0,
qHeight: Math.min(hyperCube.qSize.qcy, 10000),
qWidth: hyperCube.qSize.qcx
}]);
if (dataPages && dataPages[0]?.qMatrix) {
dataMatrix = dataPages[0].qMatrix;
console.error(`[Extract Data] Successfully fetched ${dataMatrix.length} rows`);
}
} catch (fetchError) {
console.error('[Extract Data] Error fetching data:', fetchError);
throw fetchError;
}
} else if (hyperCube.qDataPages && hyperCube.qDataPages[0]?.qMatrix) {
dataMatrix = hyperCube.qDataPages[0].qMatrix;
console.error(`[Extract Data] Using existing data: ${dataMatrix.length} rows`);
}
const dimensionCount = hyperCube.qDimensionInfo?.length || 0;
const measureCount = hyperCube.qMeasureInfo?.length || 0;
const processedData = dataMatrix.map((row: any[]) => {
const processedRow: any[] = [];
// Process dimensions
for (let i = 0; i < dimensionCount; i++) {
processedRow.push({
text: row[i]?.qText || '',
number: row[i]?.qNum,
isNull: row[i]?.qIsNull || false
});
}
// Process measures
for (let i = 0; i < measureCount; i++) {
const cellIndex = dimensionCount + i;
const cell = row[cellIndex];
processedRow.push({
text: cell?.qText || '',
number: cell?.qNum,
isNull: cell?.qIsNull || false
});
}
return processedRow;
});
return processedData;
}
private convertToObjectDataSampleUltimateFix(extractedData: any, objectId: string): ObjectDataSample {
const sampleData: Array<Record<string, any>> = [];
const columnInfo: Array<any> = [];
if (extractedData.data && Array.isArray(extractedData.data)) {
extractedData.data.forEach((row: any, rowIndex: number) => {
const simpleRow: Record<string, any> = {};
// Process dimensions
extractedData.dimensions?.forEach((dim: any, dimIndex: number) => {
const cell = row[dimIndex];
simpleRow[dim.label] = cell?.text || '';
});
// Process measures
extractedData.measures?.forEach((measure: any, measureIndex: number) => {
const cellIndex = extractedData.dimensions.length + measureIndex;
const cell = row[cellIndex];
if (cell && typeof cell.number === 'number' && !isNaN(cell.number)) {
simpleRow[measure.label] = cell.number;
} else {
simpleRow[measure.label] = cell?.text || 0;
}
});
sampleData.push(simpleRow);
});
}
// Build column info
extractedData.dimensions?.forEach((dim: any) => {
columnInfo.push({
name: dim.label,
type: 'dimension',
dataType: 'text'
});
});
extractedData.measures?.forEach((measure: any) => {
columnInfo.push({
name: measure.label,
type: 'measure',
dataType: 'numeric',
expression: measure.expression
});
});
return {
objectId,
objectType: extractedData.objectType || 'unknown',
sampleData,
totalRows: extractedData.totalRows || sampleData.length,
columnInfo,
dataExtractedAt: new Date().toISOString()
};
}
private async extractKPIDataFromObjectUltimateFix(layout: any, object: any): Promise<any> {
console.error('[KPI Extraction] Extracting KPI data');
const kpiData = {
objectType: 'kpi',
dimensions: [],
measures: [{
label: layout.title || 'KPI Value',
expression: layout.qHyperCube?.qMeasureInfo?.[0]?.qDef?.qDef || 'Unknown',
position: 0,
type: 'measure'
}],
data: [[{
text: layout.qHyperCube?.qDataPages?.[0]?.qMatrix?.[0]?.[0]?.qText || '',
number: layout.qHyperCube?.qDataPages?.[0]?.qMatrix?.[0]?.[0]?.qNum || 0,
isNull: false
}]],
totalRows: 1,
extractionMetadata: {
timestamp: new Date().toISOString(),
objectType: 'kpi',
extractionMode: 'kpi-extraction'
}
};
return kpiData;
}
// Utility methods
private mapQlikDataType(qTag?: string): 'text' | 'numeric' | 'date' | 'timestamp' | 'dual' {
if (!qTag) return 'text';
const tag = qTag.toLowerCase();
if (tag.includes('$numeric') || tag.includes('$integer')) return 'numeric';
if (tag.includes('$date')) return 'date';
if (tag.includes('$timestamp')) return 'timestamp';
if (tag.includes('$text')) return 'text';
return 'dual';
}
private mapQlikNumberFormat(qType?: string): string {
switch (qType) {
case 'F': return 'fixed';
case 'I': return 'integer';
case 'M': return 'money';
case 'D': return 'date';
case 'T': return 'time';
case 'TS': return 'timestamp';
case 'IV': return 'interval';
default: return 'number';
}
}
private formatHypercubeData(dataPage: any, hyperCube: any): any {
const matrix = dataPage.qMatrix;
const dimensions = hyperCube.qDimensionInfo || [];
const measures = hyperCube.qMeasureInfo || [];
const columns = [
...dimensions.map((dim: any, idx: number) => ({
name: dim.qFallbackTitle,
type: 'dimension',
index: idx,
field: dim.qGroupFieldDefs?.[0] || dim.qFieldDefs?.[0]
})),
...measures.map((measure: any, idx: number) => ({
name: measure.qFallbackTitle,
type: 'measure',
index: dimensions.length + idx,
expression: measure.qDef?.qDef,
format: measure.qNumFormat
}))
];
const rows = matrix.map((row: any[], rowIndex: number) => {
const rowData: any = {};
row.forEach((cell: any, cellIndex: number) => {
const column = columns[cellIndex];
if (column) {
if (column.type === 'dimension') {
rowData[column.name] = cell.qText || '';
} else {
rowData[column.name] = {
value: cell.qNum !== undefined ? cell.qNum : null,
text: cell.qText || '',
...(column.format && { format: column.format })
};
}
}
});
return rowData;
});
if (rows.length > 0) {
console.error(`📊 Sample data (first row):`, JSON.stringify(rows[0], null, 2));
}
return {
columns,
rows,
totalRows: hyperCube.qSize.qcy,
fetchedRows: rows.length,
hasMore: rows.length < hyperCube.qSize.qcy
};
}
private buildExtractionStatsWithData(sheets: SimpleSheet[]): any {
const stats = {
totalLayoutContainers: 0,
totalNestedObjects: 0,
successfulExtractions: 0,
failedExtractions: 0,
visualizationTypeBreakdown: {} as Record<string, number>,
containerAnalysis: {
totalContainers: 0,
containersWithNested: 0,
averageNestedPerContainer: 0,
maxNestingDepth: 0
},
dataExtractionStats: {
objectsWithData: 0,
totalDataRows: 0,
hypercubesExtracted: 0,
listObjectsExtracted: 0,
fieldValuesExtracted: 0,
dataExtractionErrors: 0
}
};
let maxDepth = 0;
let totalNestedObjects = 0;
sheets.forEach(sheet => {
// Count containers
stats.containerAnalysis.totalContainers += sheet.containerInfo.length;
stats.totalLayoutContainers += sheet.containerInfo.length;
// Count containers with nested objects
sheet.containerInfo.forEach(container => {
if (container.nestedObjects.length > 0) {
stats.containerAnalysis.containersWithNested++;
totalNestedObjects += container.nestedObjects.length;
}
});
// Count visualization objects
sheet.visualizationObjects.forEach(obj => {
stats.successfulExtractions++;
// Track type breakdown
const type = obj.type || 'unknown';
stats.visualizationTypeBreakdown[type] = (stats.visualizationTypeBreakdown[type] || 0) + 1;
// Track nesting depth
if (obj.nestingLevel && obj.nestingLevel > maxDepth) {
maxDepth = obj.nestingLevel;
}
// Track data extraction
if (obj.sampleData && obj.sampleData.length > 0) {
stats.dataExtractionStats.objectsWithData++;
stats.dataExtractionStats.totalDataRows += obj.sampleData.length;
stats.dataExtractionStats.hypercubesExtracted++;
}
});
});
stats.totalNestedObjects = totalNestedObjects;
stats.containerAnalysis.maxNestingDepth = maxDepth;
stats.containerAnalysis.averageNestedPerContainer =
stats.containerAnalysis.totalContainers > 0
? Math.round((totalNestedObjects / stats.containerAnalysis.totalContainers) * 100) / 100
: 0;
return stats;
}
private buildAppDataSummary(sheets: SimpleSheet[]): any {
const allDimensions = new Set<string>();
const allMeasures = new Set<string>();
const dataSources = new Set<string>();
let totalDataPoints = 0;
sheets.forEach(sheet => {
sheet.visualizationObjects.forEach(obj => {
// Collect dimensions
obj.dimensions?.forEach(dim => {
if (dim.fieldName) {
allDimensions.add(dim.fieldName);
// Try to identify data source from field name
const parts = dim.fieldName.split('.');
if (parts.length > 1) {
dataSources.add(parts[0]);
}
}
});
// Collect measures
obj.measures?.forEach(measure => {
if (measure.expression) {
allMeasures.add(measure.label);
}
});
// Count data points
if (obj.sampleData) {
totalDataPoints += obj.sampleData.length *
((obj.dimensions?.length || 0) + (obj.measures?.length || 0));
}
});
});
// Determine complexity
const uniqueObjects = sheets.reduce((sum, sheet) => sum + sheet.totalObjects, 0);
const complexity =
uniqueObjects < 10 ? 'simple' :
uniqueObjects < 50 ? 'moderate' : 'complex';
return {
totalDataPoints,
dimensionsUsed: Array.from(allDimensions),
measuresUsed: Array.from(allMeasures),
dataSources: Array.from(dataSources),
dataModelComplexity: complexity,
estimatedDataFreshness: new Date().toISOString()
};
}
}
