import { ApiClient, ResolvedUser } from '../utils/api-client';
import { CacheManager } from '../utils/cache-manager';
import { logger } from '../utils/logger.js';
// ===== TYPE DEFINITIONS =====
/**
* Dataset subcategory types
*/
type DatasetSubType = 'dataset' | 'qvd' | 'data-file' | 'dataconnection';
/**
* Main item types with dataset as a parent category
*/
type MainItemType = 'app' | 'dataset' | 'script' | 'automation' | 'ml-deployment' |
'ml-experiment' | 'assistant' | 'note' | 'link' | 'glossary' |
'knowledge-base' | 'dataflow' | 'datafilefolder' ;
// ===== INTERFACES (keeping exact same structure) =====
export interface SpaceCatalogSearchOptions {
query?: string;
spaceType?: 'personal' | 'shared' | 'managed' | 'data' | 'all';
ownerId?: string;
memberUserId?: string;
hasDataAssets?: boolean;
minItems?: number;
limit?: number;
offset?: number;
sortBy?: 'name' | 'created' | 'modified' | 'itemCount' | 'memberCount';
sortOrder?: 'asc' | 'desc';
includeMembers?: boolean;
includeItems?: boolean;
includeCounts?: boolean; // NEW: Always fetch counts without fetching items
}
export interface SpaceCatalogResult {
spaces: QlikSpace[];
totalCount: number;
searchTime: number;
facets: SpaceFacets;
summary: SpaceSummary;
}
export interface QlikSpace {
id: string;
name: string;
description?: string;
type: string;
spaceInfo: {
ownerId: string;
ownerName: string;
ownerEmail?: string;
createdDate: string;
modifiedDate: string;
isActive: boolean;
visibility: string;
};
members: SpaceMember[];
dataAssets: DataAsset[];
spaceItems: SpaceItem[];
statistics: SpaceStatistics;
}
export interface SpaceMember {
userId: string;
userName: string;
userEmail?: string;
role: 'owner' | 'admin' | 'contributor' | 'consumer' | 'viewer';
assignedDate?: string;
assignedBy?: string;
isActive: boolean;
}
export interface DataAsset {
id: string;
name: string;
type: 'dataset' | 'qvd' | 'data-file';
ownerId: string;
ownerName: string;
createdDate?: string;
modifiedDate?: string;
size?: number;
resourceType?: string;
resourceSubType?: string;
resourceAttributes?: any;
description?: string;
tags: string[];
}
export interface SpaceItem {
id: string;
name: string;
type: MainItemType; // Using the MainItemType here
ownerId: string;
ownerName: string;
createdDate?: string;
modifiedDate?: string;
size?: number;
resourceType?: string;
resourceSubType?: string;
resourceAttributes?: any;
description?: string;
tags: string[];
}
export interface SpaceStatistics {
totalItems: number;
totalMembers: number;
membersByRole: Record<string, number>;
lastActivity: string;
storageUsed: number;
itemsByType?: Record<string, number>;
// Dataset breakdown
datasetsCount?: number; // Total of all dataset subcategories
dataAssetsCount?: number; // Same as datasetsCount (for backward compatibility)
datasetsBySubType?: {
dataset: number; // Native Qlik datasets
qvd: number; // QVD files
dataFile: number; // Other data files (CSV, Excel, JSON, etc.)
};
appsCount?: number;
automationsCount?: number;
scriptsCount?: number;
mlDeploymentsCount?: number;
dataflowsCount?: number;
knowledgeBasesCount?: number;
assistantsCount?: number;
mlExperimentsCount?: number;
}
export interface SpacePermissions {
canView: boolean;
canEdit: boolean;
canDelete: boolean;
canManageMembers: boolean;
canCreateItems: boolean;
}
export interface SpaceFacets {
spaceTypes: Array<{ type: string; count: number }>;
owners: Array<{ ownerId: string; ownerName: string; count: number }>;
itemTypes: Array<{ type: string; count: number }>;
activityRanges: Array<{ range: string; count: number }>;
}
export interface SpaceSummary {
totalSpaces: number;
totalItems: number;
totalMembers: number;
totalDataAssets: number;
averageStats: {
itemsPerSpace: number;
membersPerSpace: number;
};
topSpacesByItems: Array<{ name: string; items: number; type: string }>;
topSpacesByMembers: Array<{ name: string; members: number; type: string }>;
recentActivity: Array<{ name: string; lastActivity: string; type: string }>;
}
export interface SpaceItemsOptions {
// Space selection (choose one)
spaceId?: string;
spaceIds?: string[];
allSpaces?: boolean;
skipPagination?: boolean;
// Filters
query?: string;
itemTypes?: string[];
ownerId?: string;
ownerName?: string;
ownerEmail?: string;
spaceType?: 'personal' | 'shared' | 'managed' | 'data' | 'all';
hasDataAssets?: boolean;
// Date filtering (ADD THESE IF MISSING)
startDate?: string;
endDate?: string;
dateField?: 'created' | 'modified';
timeframe?: number;
timeframeUnit?: 'hours' | 'days' | 'weeks' | 'months';
// Options
includeSpaceInfo?: boolean;
includeOwnerInfo?: boolean;
includeDetails?: boolean; // THIS SHOULD BE HERE
groupBySpace?: boolean;
groupByType?: boolean;
// Pagination
cursor?: string; // NEW: cursor for pagination (base64 encoded JSON)
limit?: number; // Max 100
offset?: number; // DEPRECATED - use cursor instead
// Sorting
sortBy?: 'name' | 'created' | 'modified' | 'size' | 'type';
sortOrder?: 'asc' | 'desc';
}
export interface SpaceItemsResult {
success: boolean;
items?: SpaceItemResult[];
groupedBySpace?: SpaceGroupResult[];
groupedByType?: TypeGroupResult[];
metadata?: {
totalItems: number;
returnedItems: number;
cursor?: string;
offset: number;
limit: number;
hasMore: boolean;
searchTime: number;
spacesSearched: number;
filters: any;
};
error?: string;
}
export interface SpaceItemResult {
id: string;
name: string;
normalizeItemType: string; // Changed from 'type' to 'normalizeItemType'
resourceType?: string;
owner?: string;
ownerId?: string;
created?: string;
modified?: string;
size?: number;
sizeFormatted?: string;
description?: string;
tags?: string[];
space?: {
id: string;
name: string;
type: string;
owner: string;
};
}
export interface SpaceGroupResult {
space: {
id: string;
name: string;
type: string;
owner: string;
};
itemCount: number;
items: SpaceItemResult[];
}
export interface TypeGroupResult {
type: string;
count: number;
items: SpaceItemResult[];
}
interface EnhancedSpaceItem extends SpaceItem {
_spaceId: string;
_spaceName: string;
_spaceType: string;
_spaceOwner: string;
_spaceOwnerId: string;
owner?: string | { id?: string; name?: string };
resourceSubType?: string; // Add this for proper type detection
}
export interface ContentStatistics {
summary: {
totalSpaces: number;
totalItems: number;
totalDataAssets: number;
totalMembers: number;
totalStorage: number;
averageItemsPerSpace: number;
averageMembersPerSpace: number;
};
typeDistribution: Record<string, {
count: number;
percentage: number;
spacesWithType: number;
uniqueOwners: number;
totalSizeFormatted?: string;
averageSize?: string;
}>;
assetTypes: string[];
detailedTable: any[][];
spaceBreakdown: {
personal: number;
shared: number;
managed: number;
data: number;
};
lastUpdated: string;
// Optional sections
dateMetrics?: {
oldestContent: any;
newestContent: any;
contentByMonth: Array<{ month: string; count: number }>;
monthsWithContent: number;
};
userMetrics?: {
totalContentOwners: number;
topContentOwners: any[];
averageItemsPerOwner: number;
};
// Grouped views
bySpace?: any[];
byType?: any[];
byOwner?: any[];
}
// ===== ENHANCED DATA CATALOG SERVICE IMPLEMENTATION =====
export class DataCatalogService {
private readonly apiClient: ApiClient;
private readonly cacheManager: CacheManager;
// Enhanced caches with better structure
private spacesCache = new Map<string, QlikSpace[]>();
private spaceDetailsCache = new Map<string, QlikSpace>();
private dataAssetSearchCache = new Map<string, any>();
private contentSearchIndex = new Map<string, Set<string>>();
// Performance tracking
private searchPerformance = {
totalSearches: 0,
cacheHits: 0,
apiCalls: 0,
averageSearchTime: 0
};
constructor(apiClient: ApiClient, cacheManager: CacheManager) {
this.apiClient = apiClient;
this.cacheManager = cacheManager;
logger.debug('[DataCatalogService] Initialized with enhanced search capabilities');
}
/**
* UNIFIED METHOD: Get spaces catalog with optional filtering
* This replaces both scanSpacesCatalog and searchSpacesCatalog
*/
async getSpacesCatalog(options: SpaceCatalogSearchOptions & {
force?: boolean;
useCache?: boolean;
includeMembers?: boolean;
includeCounts?: boolean; // NEW: Fetch just counts, not full data
getAllSpaces?: boolean;
maxSpaces?: number;
} = {}): Promise<SpaceCatalogResult> {
const startTime = Date.now();
this.searchPerformance.totalSearches++;
// Determine if this is a search or scan based on presence of filters
const isSearch = !!(options.query || options.spaceType || options.ownerId ||
options.memberUserId || options.hasDataAssets);
const {
force = false,
useCache = true,
includeMembers = true, // false for search, true for scan
includeCounts = true, // NEW: Default to true - always get counts!
getAllSpaces = false,
maxSpaces,
...filterOptions
} = options;
// Log what we're doing
logger.debug(`[SpacesCatalog] Operation: ${isSearch ? 'search' : 'scan'}, ` +
`includeCounts: ${includeCounts}`);
try {
// Determine if we need all spaces
const needsAllSpaces = getAllSpaces || maxSpaces !== undefined;
// Adjust cache key to include new flags
const cacheKey = this.generateCacheKey({
...options,
getAllSpaces: needsAllSpaces,
effectiveLimit: needsAllSpaces ? (maxSpaces || 'all') : options.limit
});
// Check cache first (unless force refresh)
if (!force && useCache && this.spacesCache.has(cacheKey)) {
logger.debug('[SpacesCatalog] Returning cached results');
this.searchPerformance.cacheHits++;
const cachedSpaces = this.spacesCache.get(cacheKey);
// FIX 1: Handle undefined cache result
if (cachedSpaces) {
const searchTime = Date.now() - startTime;
return this.buildCatalogResult(cachedSpaces, cachedSpaces.length, searchTime);
}
// If cache returns undefined, continue to fetch
}
let allSpaces: QlikSpace[];
// NEW: If getAllSpaces or maxSpaces is set, fetch ALL spaces with pagination
if (needsAllSpaces) {
logger.debug(`[SpacesCatalog] Fetching ALL spaces (maxSpaces: ${maxSpaces || 'unlimited'})`);
allSpaces = await this.fetchAllSpacesWithPagination({
...filterOptions,
targetCount: maxSpaces,
includeMembers
});
}
// Original logic for filtered/full fetch
else if (options.query || options.spaceType || options.ownerId) {
logger.debug('[SpacesCatalog] Using filtered search strategy');
allSpaces = await this.fetchFilteredSpaces(options);
}
else {
logger.debug('[SpacesCatalog] Using full catalog scan strategy');
allSpaces = await this.fetchAllSpaces(options);
}
// Apply client-side filters if needed
let filteredSpaces = allSpaces;
// Filter by member
if (options.memberUserId) {
filteredSpaces = filteredSpaces.filter(space =>
space.members?.some(member => member.userId === options.memberUserId)
);
}
// Filter by data assets
if (options.hasDataAssets !== undefined) {
filteredSpaces = filteredSpaces.filter(space =>
options.hasDataAssets ?
(space.dataAssets?.length ?? 0) > 0 :
(space.dataAssets?.length ?? 0) === 0
);
}
// Filter by minimum items - FIX 2: Handle undefined minItems
if (options.minItems !== undefined && options.minItems !== null) {
const minItemsValue = options.minItems; // TypeScript now knows this is defined
filteredSpaces = filteredSpaces.filter(space =>
(space.statistics?.totalItems ?? 0) >= minItemsValue
);
}
// Apply sorting
if (options.sortBy) {
filteredSpaces = this.sortSpaces(filteredSpaces, options.sortBy, options.sortOrder);
}
// Apply pagination (skip if getAllSpaces was used as we already have the right amount)
const totalBeforePagination = filteredSpaces.length;
let paginatedSpaces = filteredSpaces;
if (!needsAllSpaces) {
const offset = options.offset || 0;
const limit = options.limit || 100;
paginatedSpaces = filteredSpaces.slice(offset, offset + limit);
}
// Cache the results
if (useCache) {
this.spacesCache.set(cacheKey, paginatedSpaces);
}
// Update performance metrics
const searchTime = Date.now() - startTime;
this.updatePerformanceMetrics(searchTime);
// Build and return the result
return this.buildCatalogResult(paginatedSpaces, totalBeforePagination, searchTime);
}
catch (error) {
console.error('[SpacesCatalog] Failed to get spaces catalog:', error);
throw error;
}
}
// NEW: Helper method to fetch all spaces with proper pagination
private async fetchAllSpacesWithPagination(options: {
targetCount?: number;
includeMembers?: boolean;
includeItems?: boolean;
query?: string;
spaceType?: string;
ownerId?: string;
}): Promise<QlikSpace[]> {
const allSpaces: any[] = [];
let offset = 0;
const batchSize = 100; // API max limit
let hasMore = true;
const targetCount = options.targetCount || Infinity;
logger.debug(`[SpacesCatalog] Starting paginated fetch (target: ${targetCount === Infinity ? 'all' : targetCount})`);
while (hasMore && allSpaces.length < targetCount) {
// Build URL with filters
const params = new URLSearchParams();
params.set('limit', String(batchSize));
params.set('offset', String(offset));
// Apply filters at API level when possible
if (options.query) params.set('query', options.query);
if (options.ownerId) params.set('ownerId', options.ownerId);
if (options.spaceType && options.spaceType !== 'all') {
params.set('type', options.spaceType);
}
const url = `/api/v1/spaces?${params.toString()}`;
logger.debug(`[SpacesCatalog] Fetching batch: offset=${offset}, limit=${batchSize}`);
const response = await this.apiClient.makeRequest(url, 'GET');
const batch = response.data || response || [];
if (!Array.isArray(batch) || batch.length === 0) {
hasMore = false;
logger.debug(`[SpacesCatalog] No more spaces found`);
} else {
// Add to collection
const remaining = targetCount - allSpaces.length;
const toAdd = batch.slice(0, remaining);
allSpaces.push(...toAdd);
logger.debug(`[SpacesCatalog] Retrieved ${batch.length} spaces, total: ${allSpaces.length}`);
// Check if we have more to fetch
offset += batch.length;
hasMore = batch.length === batchSize && allSpaces.length < targetCount;
}
}
logger.debug(`[SpacesCatalog] Pagination complete. Total spaces: ${allSpaces.length}`);
// Enrich spaces if needed
if (options.includeMembers || options.includeItems) {
logger.debug(`[SpacesCatalog] Enriching ${allSpaces.length} spaces...`);
return await this.enrichSpaces(allSpaces, {
includeMembers: options.includeMembers,
includeItems: options.includeItems
});
}
return allSpaces;
}
/**
* Fetch all spaces with comprehensive pagination
*/
private async fetchAllSpaces(options: Partial<SpaceCatalogSearchOptions>): Promise<QlikSpace[]> {
logger.debug('[SpacesCatalog] Fetching all spaces...');
this.searchPerformance.apiCalls++;
const allSpaces: any[] = [];
let offset = 0;
const limit = 100; // API max
let hasMore = true;
while (hasMore) {
const spacesUrl = `/api/v1/spaces?limit=${limit}&offset=${offset}`;
logger.debug(`[SpacesCatalog] Fetching spaces: offset=${offset}, limit=${limit}`);
const response = await this.apiClient.makeRequest(spacesUrl, 'GET');
const spaces = response.data || response || [];
if (spaces.length === 0) {
hasMore = false;
} else {
allSpaces.push(...spaces);
hasMore = spaces.length === limit;
offset += spaces.length;
}
}
logger.debug(`[SpacesCatalog] Found ${allSpaces.length} total spaces`);
// Enrich spaces with details
return this.enrichSpaces(allSpaces, options);
}
/**
* Fetch filtered spaces using API query parameters
*/
private async fetchFilteredSpaces(options: SpaceCatalogSearchOptions): Promise<QlikSpace[]> {
logger.debug('[SpacesCatalog] Fetching filtered spaces...');
this.searchPerformance.apiCalls++;
let allMatchingSpaces: any[] = [];
// If there's a multi-word query, try different search strategies
if (options.query && options.query.trim().split(/\s+/).length > 1) {
logger.debug('[SpacesCatalog] Multi-word query detected, trying multiple strategies...');
// Strategy 1: Try exact query first
allMatchingSpaces = await this.fetchSpacesWithQuery(options, options.query);
// Strategy 2: If no results, try individual tokens
if (allMatchingSpaces.length === 0) {
logger.debug('[SpacesCatalog] No results with exact query, trying individual tokens...');
const tokens = options.query.toLowerCase().split(/\s+/).filter(t => t.length > 0);
for (const token of tokens) {
const tokenResults = await this.fetchSpacesWithQuery(options, token);
// Add unique results
for (const space of tokenResults) {
if (!allMatchingSpaces.find(s => s.id === space.id)) {
allMatchingSpaces.push(space);
}
}
}
}
} else {
// Single word query or other filters
allMatchingSpaces = await this.fetchSpacesWithQuery(options, options.query);
}
logger.debug(`[SpacesCatalog] Found ${allMatchingSpaces.length} matching spaces`);
// Enrich spaces with details
const enrichedSpaces = await this.enrichSpaces(allMatchingSpaces, options);
// If we had a multi-word query, apply client-side filtering to ensure relevance
if (options.query && options.query.trim().split(/\s+/).length > 1) {
return this.performTokenBasedSearch(enrichedSpaces, options.query);
}
return enrichedSpaces;
}
/**
* Fetch spaces with a specific query
*/
private async fetchSpacesWithQuery(options: SpaceCatalogSearchOptions, query?: string): Promise<any[]> {
const allSpaces: any[] = [];
let offset = 0;
const limit = 100;
let hasMore = true;
while (hasMore) {
const params = this.buildApiQueryParams({ ...options, query }, offset, limit);
const spacesUrl = `/api/v1/spaces?${params.toString()}`;
const response = await this.apiClient.makeRequest(spacesUrl, 'GET');
const spaces = response.data || response || [];
if (spaces.length === 0) {
hasMore = false;
} else {
allSpaces.push(...spaces);
hasMore = spaces.length === limit;
offset += spaces.length;
}
}
return allSpaces;
}
/**
* Perform token-based search on enriched spaces
*/
private performTokenBasedSearch(spaces: QlikSpace[], query: string): QlikSpace[] {
const queryLower = query.toLowerCase();
const queryTokens = queryLower.split(/[\s,._\-\/]+/).filter(t => t.length > 0);
// Score each space based on token matches
const scoredSpaces = spaces.map(space => {
let score = 0;
let matchedTokens = 0;
// Create searchable content
const searchableContent = [
space.name,
space.description,
space.spaceInfo.ownerName,
...space.members.map(m => m.userName),
...space.dataAssets.map(d => d.name),
...space.spaceItems.map(i => i.name)
].filter(Boolean).join(' ').toLowerCase();
// Check each token
for (const token of queryTokens) {
if (searchableContent.includes(token)) {
matchedTokens++;
// Higher score for matches in space name
if (space.name.toLowerCase().includes(token)) {
score += 10;
}
// Medium score for description
else if (space.description?.toLowerCase().includes(token)) {
score += 5;
}
// Lower score for other matches
else {
score += 1;
}
}
}
// Bonus for matching all tokens
if (matchedTokens === queryTokens.length) {
score += 20;
}
return { space, score, matchedTokens };
});
// Filter and sort by score
return scoredSpaces
.filter(item => item.matchedTokens > 0)
.sort((a, b) => b.score - a.score)
.map(item => item.space);
}
/**
* Build API query parameters for filtered searches
*/
private buildApiQueryParams(options: SpaceCatalogSearchOptions, offset: number, limit: number): URLSearchParams {
const params = new URLSearchParams();
// Pagination
params.append('limit', limit.toString());
params.append('offset', offset.toString());
// Search query
if (options.query) {
params.append('name', options.query);
}
// Space type filter
if (options.spaceType && options.spaceType !== 'all') {
params.append('type', options.spaceType);
}
// Owner filter
if (options.ownerId) {
params.append('ownerId', options.ownerId);
}
// Sorting
if (options.sortBy) {
const sortPrefix = options.sortOrder === 'desc' ? '-' : '+';
const sortField = this.mapSortField(options.sortBy);
params.append('sort', `${sortPrefix}${sortField}`);
}
return params;
}
/**
* Map our sort fields to API fields
*/
private mapSortField(sortBy: string): string {
const mapping: Record<string, string> = {
'created': 'createdAt',
'modified': 'updatedAt',
'name': 'name',
'itemCount': 'name', // API doesn't support, will sort client-side
'memberCount': 'name' // API doesn't support, will sort client-side
};
return mapping[sortBy] || 'name';
}
/**
* Enrich spaces with detailed information
*/
/**
* Enrich spaces with detailed information
*/
private async enrichSpaces(
spaces: any[],
options: Partial<SpaceCatalogSearchOptions>
): Promise<QlikSpace[]> {
if (spaces.length === 0) return [];
logger.debug(`[SpacesCatalog] Enriching ${spaces.length} spaces`);
// Owner ID'lerini topla ve resolve et
const ownerIds = [...new Set(spaces
.map(s => s.ownerId || s.owner?.id)
.filter(Boolean))] as string[];
const resolvedUsers = ownerIds.length > 0 ?
await this.apiClient.resolveOwnersToUsers(ownerIds) :
new Map<string, ResolvedUser>();
// Her space için paralel işlem
const enrichedSpaces = await Promise.all(
spaces.map(async (spaceInfo) => {
try {
const shouldFetchMembers = options.includeMembers !== false;
const shouldFetchItems = options.includeItems === true;
// Full detail gerekiyorsa (members listesi için)
if (shouldFetchMembers || shouldFetchItems) {
const detailedSpace = await this.getSpaceDetails(
spaceInfo.id,
shouldFetchMembers,
shouldFetchItems
);
return detailedSpace || this.createBasicSpace(spaceInfo, resolvedUsers);
}
// Sadece basic space + counts
let basicSpace = this.createBasicSpace(spaceInfo, resolvedUsers);
// Count'ları al (hem item hem member)
const counts = await this.getSpaceCounts(spaceInfo.id);
basicSpace.statistics.totalItems = counts.itemCount;
basicSpace.statistics.totalMembers = counts.memberCount;
basicSpace.statistics.storageUsed = 0;
return basicSpace;
} catch (error) {
console.error(`[SpacesCatalog] Failed to enrich space ${spaceInfo.id}:`, error);
return this.createBasicSpace(spaceInfo, resolvedUsers);
}
})
);
return enrichedSpaces;
}
/**
* Create basic space with actual counts (not full details)
*/
private createBasicSpaceWithCounts(
spaceInfo: any,
resolvedUsers: Map<string, ResolvedUser>,
counts: { itemCount: number; memberCount: number }
): QlikSpace {
const space = this.createBasicSpace(spaceInfo, resolvedUsers);
// Update statistics with actual counts
space.statistics.totalItems = counts.itemCount;
space.statistics.totalMembers = counts.memberCount;
return space;
}
/**
* Create fully typed space from API response
*/
private async createFullSpace(
spaceInfo: any,
includeMembers: boolean,
includeItems: boolean
): Promise<QlikSpace> {
try {
// Owner resolve
const ownerId = spaceInfo.ownerId || spaceInfo.owner?.id;
const resolvedUsers = ownerId ?
await this.apiClient.resolveOwnersToUsers([ownerId]) :
new Map<string, ResolvedUser>();
const ownerUser = ownerId ? resolvedUsers.get(ownerId) : null;
// Members fetch ve resolve
const members = includeMembers ?
await this.fetchSpaceMembers(spaceInfo.id) : [];
// Items fetch (eğer isteniyorsa)
let spaceItems: any[] = [];
let dataAssets: any[] = [];
if (includeItems) {
logger.debug(`[SpacesCatalog] Fetching items for space ${spaceInfo.id}`);
const fetchedItems = await this.fetchAllSpaceItems(spaceInfo.id);
spaceItems = fetchedItems.items || [];
dataAssets = fetchedItems.dataAssets || [];
}
// Count'ları al (items fetch edilmese bile)
const counts = await this.getSpaceCounts(spaceInfo.id);
// Member role breakdown
const membersByRole: Record<string, number> = {};
members.forEach((member: any) => {
const role = member.roles?.[0] || member.role || 'member';
membersByRole[role] = (membersByRole[role] || 0) + 1;
});
// Statistics
const statistics: SpaceStatistics = {
totalItems: counts.itemCount, // getSpaceCounts'tan
itemsByType: {},
totalMembers: counts.memberCount, // getSpaceCounts'tan
membersByRole: membersByRole,
lastActivity: spaceInfo.updatedAt || spaceInfo.createdAt || '',
storageUsed: 0
};
return {
id: spaceInfo.id,
name: spaceInfo.name || 'Unnamed Space',
description: spaceInfo.description || '',
type: spaceInfo.type || 'shared',
spaceInfo: {
ownerId: ownerId,
ownerName: ownerUser?.displayName || 'Unknown Owner',
ownerEmail: ownerUser?.email || '',
createdDate: spaceInfo.createdAt || '',
modifiedDate: spaceInfo.updatedAt || '',
isActive: true,
visibility: 'private'
},
members: members,
dataAssets: dataAssets,
spaceItems: spaceItems,
statistics: statistics
};
} catch (error) {
console.error('[SpacesCatalog] Failed to create full space:', error);
throw error;
}
}
/**
* Create basic space with user resolution
*/
private createBasicSpace(
spaceInfo: any,
resolvedUsers: Map<string, ResolvedUser>
): QlikSpace {
const ownerId = spaceInfo.ownerId || spaceInfo.owner?.id;
const ownerUser = ownerId ? resolvedUsers.get(ownerId) : null;
// Use -1 to indicate "not fetched" - this tells Claude/users that we don't know the count
const basicStats = {
totalItems: -1, // -1 means "not fetched", not empty
itemsByType: {},
totalMembers: -1, // -1 means "not fetched", not empty
membersByRole: {},
lastActivity: spaceInfo.updatedAt || spaceInfo.createdAt,
storageUsed: -1
};
return {
id: spaceInfo.id,
name: spaceInfo.name,
description: spaceInfo.description || '',
type: spaceInfo.type || 'unknown',
spaceInfo: {
ownerId: ownerId || 'unknown',
ownerName: ownerUser?.displayName || 'Unknown',
ownerEmail: ownerUser?.email || 'unknown@example.com',
createdDate: spaceInfo.createdAt || new Date().toISOString(),
modifiedDate: spaceInfo.updatedAt || new Date().toISOString(),
isActive: true,
visibility: spaceInfo.visibility || 'private'
},
members: [],
dataAssets: [],
spaceItems: [],
statistics: basicStats
};
}
/**
* Normalize role with better mapping
*/
private normalizeRole(role: string | string[]): 'owner' | 'admin' | 'contributor' | 'consumer' | 'viewer' {
const roleStr = Array.isArray(role) ? role[0] : role;
const normalizedRole = (roleStr || 'consumer').toLowerCase();
if (normalizedRole.includes('owner') || normalizedRole.includes('spaceadmin')) {
return 'owner';
} else if (normalizedRole.includes('admin') || normalizedRole.includes('can_edit')) {
return 'admin';
} else if (normalizedRole.includes('contributor') || normalizedRole.includes('can_contribute')) {
return 'contributor';
} else if (normalizedRole.includes('view') || normalizedRole.includes('can_view')) {
return 'viewer';
} else {
return 'consumer';
}
}
private normalizeItemType(
resourceType?: string,
itemName?: string,
resourceSubType?: string,
resourceAttributes?: any
): MainItemType {
const type = (resourceType || '').toLowerCase();
const subType = (resourceSubType || '').toLowerCase();
const name = (itemName || '').toLowerCase();
// HANDLE QIX-DF DATAASSETS (Qlik app data) - These ARE datasets
if (type === 'dataasset' && subType === 'qix-df') {
return 'dataset'; // This is a dataset from a Qlik app
}
// Check if it's an app with a specific subtype
if (type === 'app') {
// Better script detection - check name patterns too
if (subType === 'script' ||
subType === 'load-script' ||
name.includes('script') ||
name.includes('load') ||
name.endsWith('.qvs')) {
return 'script';
}
if (subType === 'dataflow-prep' || subType === 'dataflow') {
return 'dataflow';
}
return 'app';
}
// Apps (non-app types)
if (type === 'qlik-app' || type === 'qvapp') {
return 'app';
}
// DATA CONNECTIONS/FOLDERS - Check for dataasset with qix-df and dataSetCount = 0
if (type === 'dataasset' && subType === 'qix-df' && resourceAttributes?.dataSetCount === 0) {
return 'datafilefolder';
}
// All data-related items return 'dataset' as main type
if (this.isDatasetType(type, name)) {
return 'dataset';
}
// Documentation
if (type === 'note' || type === 'notes') {
return 'note';
}
if (type === 'link' || type === 'url') {
return 'link';
}
if (type === 'glossary') {
return 'glossary';
}
// Automation and ML
if (type === 'automation' || type === 'qlik-automation' || type === 'automationscript') {
return 'automation';
}
// AutoML types
if (type === 'ml-deployment' || type === 'mldeployment' || type === 'automl-deployment') {
return 'ml-deployment';
}
if (type === 'ml-experiment' || type === 'mlexperiment' || type === 'automl-experiment') {
return 'ml-experiment';
}
// AI/Assistant
if (type === 'assistant' || type === 'qlik-assistant' || type === 'answer') {
return 'assistant';
}
// Knowledge bases
if (type === 'knowledgebase' || type === 'knowledge-base' || type === 'kb') {
return 'knowledge-base';
}
// Default - unknown types
logger.debug(`[normalizeItemType] Unknown type: ${type}, subType: ${subType} for item: ${name}`);
return 'datafilefolder'; // Default unknown files to dataset category
}
private getDatasetSubType(resourceType: string, itemName?: string): DatasetSubType {
const type = (resourceType || '').toLowerCase();
const name = (itemName || '').toLowerCase();
// QVD files (Qlik's native format)
if (type === 'qvd' || name.endsWith('.qvd')) {
return 'qvd';
}
// Native Qlik datasets/datasources AND data connections
if (type === 'dataset' || type === 'datasource' || type === 'dataconnection') {
// Check if it's actually a file with extension
if (this.hasDataFileExtension(name)) {
return 'data-file';
}
return 'dataset'; // Data connections will be categorized as 'dataset'
}
// All other data files
if (this.hasDataFileExtension(name) || this.isDataFileType(type)) {
return 'data-file';
}
// Default to dataset if we can't determine
return 'dataset';
}
/**
* Check if type/name represents any dataset type
*/
private isDatasetType(type: string, name: string): boolean {
// Check common data types - EXCLUDING dataconnection
const dataTypes = [
'dataset', 'datasource', 'qvd', 'csv', 'xlsx', 'excel',
'parquet', 'json', 'xml', 'txt', 'tsv', 'pdf', 'docx'
];
if (dataTypes.some(t => type.includes(t))) {
return true;
}
// Check file extensions
return this.hasDataFileExtension(name);
}
/**
* Check if filename has a data file extension
*/
private hasDataFileExtension(name: string): boolean {
const dataExtensions = [
'.qvd', '.csv', '.xlsx', '.xls', '.parquet', '.json', '.xml',
'.txt', '.tsv', '.dat', '.pdf', '.docx', '.doc'
];
return dataExtensions.some(ext => name.endsWith(ext));
}
/**
* Check if type represents a data file
*/
private isDataFileType(type: string): boolean {
const fileTypes = [
'dataset','csv', 'xlsx', 'excel', 'parquet', 'json', 'xml',
'txt', 'tsv', 'pdf', 'docx'
];
return fileTypes.some(t => type.includes(t));
}
// private calculateStatistics(
// filteredSpaceItems: any[],
// members: any[],
// spaceInfo: any
// ): SpaceStatistics {
// // REMOVED the filter that excluded 'dataconnection'
// const itemsToCount = filteredSpaceItems; // Now counts all items including dataconnections
// // Initialize counts
// const itemsByType: Record<string, number> = {};
// const datasetsBySubType = {
// dataset: 0,
// qvd: 0,
// dataFile: 0
// };
// // Count items
// itemsToCount.forEach((item: any) => {
// const normalizedType = this.normalizeItemType(
// item.resourceType || item.type,
// item.name,
// item.resourceSubType
// );
// // Increment main type count
// itemsByType[normalizedType] = (itemsByType[normalizedType] || 0) + 1;
// // If it's a dataset type, also track subtype
// if (normalizedType === 'dataset') {
// const subType = this.getDatasetSubType(
// item.resourceType || item.type,
// item.name
// );
// switch(subType) {
// case 'qvd':
// datasetsBySubType.qvd++;
// break;
// case 'data-file':
// datasetsBySubType.dataFile++;
// break;
// default:
// datasetsBySubType.dataset++;
// }
// }
// });
// // Count member roles
// const membersByRole: Record<string, number> = {};
// members.forEach((member: any) => {
// const role = this.normalizeRole(member.roles || member.role || 'consumer');
// membersByRole[role] = (membersByRole[role] || 0) + 1;
// });
// const datasetsTotal = itemsByType['dataset'] || 0;
// return {
// totalItems: itemsToCount.length,
// itemsByType: itemsByType,
// // Dataset counts - INCLUDING dataAssetsCount
// datasetsCount: datasetsTotal,
// dataAssetsCount: datasetsTotal, // THIS WAS MISSING!
// datasetsBySubType: datasetsBySubType,
// // Other counts
// totalMembers: members.length,
// membersByRole: membersByRole,
// lastActivity: this.getLastActivity(filteredSpaceItems, members, spaceInfo),
// storageUsed: filteredSpaceItems.reduce((sum: number, item: any) => sum + (item.size || 0), 0),
// appsCount: filteredSpaceItems.filter((item: any) => {
// const type = this.normalizeItemType(
// item.resourceType || item.type,
// item.name,
// item.resourceSubType // MUST PASS resourceSubType HERE
// );
// return type === 'app';
// }).length,
// automationsCount: itemsByType['automation'] || 0,
// scriptsCount: itemsByType['script'] || 0,
// mlDeploymentsCount: itemsByType['ml-deployment'] || 0,
// dataflowsCount: itemsByType['dataflow'] || 0,
// knowledgeBasesCount: itemsByType['knowledge-base'] || 0,
// assistantsCount: itemsByType['assistant'] || 0 ,
// mlExperimentsCount: itemsByType['ml-experiment'] || 0
// };
// }
/**
* Get last activity with space info
*/
private getLastActivity(items: any[], members: any[], spaceInfo?: any): string {
const dates = [
spaceInfo?.modifiedDate || spaceInfo?.modified,
...items.map(item => item.modifiedDate || item.modified || item.created),
...members.map(member => member.assignedDate || member.created)
].filter(Boolean);
if (dates.length === 0) return new Date().toISOString();
return dates.sort((a, b) => new Date(b).getTime() - new Date(a).getTime())[0];
}
/**
* Sort spaces by specified field
*/
private sortSpaces(spaces: QlikSpace[], sortBy?: string, sortOrder?: string): QlikSpace[] {
const order = sortOrder === 'desc' ? -1 : 1;
return [...spaces].sort((a: QlikSpace, b: QlikSpace) => {
switch (sortBy) {
case 'created':
return order * (new Date(a.spaceInfo.createdDate).getTime() - new Date(b.spaceInfo.createdDate).getTime());
case 'modified':
return order * (new Date(a.spaceInfo.modifiedDate).getTime() - new Date(b.spaceInfo.modifiedDate).getTime());
case 'itemCount':
return order * (a.statistics.totalItems - b.statistics.totalItems);
case 'memberCount':
return order * (a.statistics.totalMembers - b.statistics.totalMembers);
case 'name':
default:
return order * a.name.localeCompare(b.name);
}
});
}
private buildFacets(spaces: QlikSpace[]): SpaceFacets {
const spaceTypeCount = this.groupBy(spaces, 'type');
const ownerCount = new Map<string, { count: number; name: string }>();
spaces.forEach((space: QlikSpace) => {
const ownerId = space.spaceInfo.ownerId;
if (!ownerCount.has(ownerId)) {
ownerCount.set(ownerId, { count: 0, name: space.spaceInfo.ownerName });
}
ownerCount.get(ownerId)!.count++;
});
// itemsByType artık optional, kontrol et
const itemTypeCounts: Record<string, number> = {};
spaces.forEach((space: QlikSpace) => {
// Optional field kontrolü
if (space.statistics.itemsByType) {
Object.entries(space.statistics.itemsByType).forEach(([type, count]) => {
itemTypeCounts[type] = (itemTypeCounts[type] || 0) + count;
});
}
});
const now = new Date();
const dayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000);
const weekAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
const monthAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000);
const activityRanges = {
'last24Hours': 0,
'lastWeek': 0,
'lastMonth': 0,
'older': 0
};
spaces.forEach((space: QlikSpace) => {
const lastActivity = new Date(space.statistics.lastActivity);
if (lastActivity > dayAgo) activityRanges.last24Hours++;
else if (lastActivity > weekAgo) activityRanges.lastWeek++;
else if (lastActivity > monthAgo) activityRanges.lastMonth++;
else activityRanges.older++;
});
return {
spaceTypes: Object.entries(spaceTypeCount).map(([type, count]) => ({ type, count })),
owners: Array.from(ownerCount.entries())
.map(([ownerId, data]) => ({ ownerId, ownerName: data.name, count: data.count }))
.sort((a, b) => b.count - a.count)
.slice(0, 10),
itemTypes: Object.entries(itemTypeCounts).map(([type, count]) => ({ type, count })),
activityRanges: Object.entries(activityRanges).map(([range, count]) => ({ range, count }))
};
}
private buildSummary(allSpaces: QlikSpace[]): SpaceSummary {
return {
totalSpaces: allSpaces.length,
totalItems: allSpaces.reduce((sum, space) => sum + space.statistics.totalItems, 0),
totalMembers: allSpaces.reduce((sum, space) => sum + space.statistics.totalMembers, 0),
totalDataAssets: allSpaces.reduce((sum, space) =>
sum + (space.statistics.dataAssetsCount || 0), 0), // <-- || 0 ekle
averageStats: {
itemsPerSpace: allSpaces.length > 0
? allSpaces.reduce((sum, space) => sum + space.statistics.totalItems, 0) / allSpaces.length
: 0,
membersPerSpace: allSpaces.length > 0
? allSpaces.reduce((sum, space) => sum + space.statistics.totalMembers, 0) / allSpaces.length
: 0
},
topSpacesByItems: allSpaces
.sort((a, b) => b.statistics.totalItems - a.statistics.totalItems)
.slice(0, 10)
.map(space => ({
name: space.name,
items: space.statistics.totalItems,
type: space.type
})),
topSpacesByMembers: allSpaces
.sort((a, b) => b.statistics.totalMembers - a.statistics.totalMembers)
.slice(0, 10)
.map(space => ({
name: space.name,
members: space.statistics.totalMembers,
type: space.type
})),
recentActivity: allSpaces
.sort((a, b) => new Date(b.statistics.lastActivity).getTime() - new Date(a.statistics.lastActivity).getTime())
.slice(0, 10)
.map(space => ({
name: space.name,
lastActivity: space.statistics.lastActivity,
type: space.type
}))
};
}
/**
* Get detailed information about a dataset
* @param datasetId - The resourceId of the dataset (from qlik_search results)
*/
async getDatasetDetails(datasetId: string): Promise<any> {
try {
logger.debug(`[DataCatalogService] Getting dataset details for datasetId: ${datasetId}`);
// Step 1: Get the dataset details directly using datasetId (which is resourceId)
const datasetResponse = await this.apiClient.makeRequest(`/api/v1/data-sets/${datasetId}`, 'GET');
const dataset = datasetResponse.data || datasetResponse;
if (!dataset) {
throw new Error(`Dataset not found: ${datasetId}`);
}
logger.debug(`[DataCatalogService] Dataset found: ${dataset.name}`);
// Step 2: Find the corresponding item entry to get space info and other metadata
// Search items by resourceId to find the item entry
let item: any = null;
try {
const itemsResponse = await this.apiClient.makeRequest(
`/api/v1/items?resourceId=${datasetId}&resourceType=dataset`,
'GET'
);
const items = itemsResponse.data || itemsResponse;
if (Array.isArray(items) && items.length > 0) {
item = items[0];
logger.debug(`[DataCatalogService] Found item entry: ${item.name}, itemId: ${item.id}`);
}
} catch (itemError) {
logger.debug(`[DataCatalogService] Could not find item entry for dataset, continuing without it`);
}
// Step 3: Determine item type from dataset or item
const itemType = item ? this.normalizeItemType(
item.resourceType || item.type,
item.name,
(item as any).resourceSubType,
(item as any).resourceAttributes
) : 'dataset';
// Step 4: Initialize variables for space and connection info
let spaceName = null;
let spaceTitle = null;
let spaceType = null;
let connectionName = null;
let connectionType = null;
let isConnectionBased = false;
// Step 5: Resolve space name if we have spaceId
const spaceId = item?.spaceId || dataset.spaceId;
if (spaceId) {
try {
logger.debug(`[DataCatalogService] Fetching space details for spaceId: ${spaceId}`);
const spaceResponse = await this.apiClient.getSpace(spaceId);
if (spaceResponse) {
spaceName = spaceResponse.name;
spaceTitle = spaceResponse.title || spaceResponse.name;
spaceType = spaceResponse.type;
logger.debug(`[DataCatalogService] Resolved space name: ${spaceName}`);
}
} catch (spaceError) {
console.error(`[DataCatalogService] Failed to get space details:`, spaceError);
// Try alternative method - search for the space
try {
const searchResult = await this.getSpacesCatalog({ limit: 100 });
const matchingSpace = searchResult.spaces.find(s => s.id === spaceId);
if (matchingSpace) {
spaceName = matchingSpace.name;
spaceTitle = matchingSpace.name;
spaceType = matchingSpace.type;
logger.debug(`[DataCatalogService] Found space via search: ${spaceName}`);
}
} catch (searchError) {
console.error(`[DataCatalogService] Failed to search for space:`, searchError);
}
}
}
// Step 6: Determine if dataset is connection-based and get connection info
const hasConnectionId = dataset.createdByConnectionId || dataset.connectionId;
let appType = null;
if (hasConnectionId) {
isConnectionBased = true;
logger.debug(`[DataCatalogService] Dataset is connection-based: ${hasConnectionId}`);
// Determine app type from dataset attributes
if (dataset.schemaType === 'qix' || dataset.type === 'QixDataSet') {
appType = 'qlik';
connectionName = 'Qlik App Data Connection';
connectionType = 'qix';
} else {
// It's a database or external connection
connectionType = dataset.connectionType || dataset.sourceType || 'unknown';
// Get connection details
try {
const connectionResponse = await this.apiClient.makeRequest(
`/api/v1/data-connections/${hasConnectionId}`,
'GET'
);
const connection = connectionResponse.data || connectionResponse;
connectionName = connection.qName;
if (!connectionName) {
console.error(`[DataCatalogService] Warning: No qName found for connection ${hasConnectionId}, using name as fallback`);
connectionName = connection.name;
}
connectionType = connection.qType || connection.type || connectionType;
logger.debug(`[DataCatalogService] Resolved connection: ${connectionName} (${connectionType})`);
} catch (connError) {
console.error(`[DataCatalogService] Failed to get connection details:`, connError);
connectionName = `Connection ${hasConnectionId}`;
}
}
}
// Step 7: Build the complete dataset object
const completeDataset = {
// Item information (from items API, may be null)
itemId: item?.id,
itemName: item?.name || dataset.name,
itemType: itemType,
spaceId: spaceId,
spaceName: spaceName,
spaceTitle: spaceTitle,
spaceType: spaceType,
// Dataset information (primary - from data-sets API)
datasetId: datasetId, // The resourceId passed in
resourceId: datasetId, // Explicit for clarity
name: dataset.name || item?.name,
description: dataset.description || item?.description,
// Connection information
isConnectionBased: isConnectionBased,
connectionName: connectionName,
connectionType: connectionType,
connectionId: dataset.createdByConnectionId,
appType: appType,
// Technical details
size: dataset.size || item?.size,
rowCount: dataset.rowCount,
columnCount: dataset.columnCount || dataset.fieldCount,
sizeFormatted: this.formatFileSize(dataset.operational?.size || dataset.size || item?.size || 0),
// Schema information
fields: dataset.fields || dataset.columns || [],
schema: dataset.schema,
// Technical name for database sources
technicalName: dataset.qName || dataset.technicalName || dataset.qualifiedName,
// For BigQuery specifically, might need to construct from parts
projectId: dataset.projectId,
datasetName: dataset.datasetName,
tableName: dataset.tableName || dataset.name,
// Metadata
createdDate: dataset.createdDate || item?.createdDate,
modifiedDate: dataset.modifiedDate || item?.modifiedDate,
lastReloadDate: dataset.lastReloadDate,
createdBy: dataset.createdBy,
modifiedBy: dataset.modifiedBy,
// Full raw responses for reference
rawItem: item,
rawDataset: dataset
};
return completeDataset;
} catch (error) {
console.error('[DataCatalogService] Failed to get dataset details:', error);
throw error;
}
}
/**
* Analyze dataset connections for load script generation
*/
async analyzeDatasetConnection(dataset: any): Promise<any> {
const connectionInfo: any = {
itemId: dataset.itemId,
itemName: dataset.itemName,
spaceName: dataset.spaceName,
isConnectionBased: dataset.isConnectionBased
};
if (dataset.isConnectionBased && dataset.appType === 'qlik') {
// This is a QIX dataset from another Qlik app
connectionInfo.sourceType = 'qlik_app';
connectionInfo.connectionType = 'binary';
connectionInfo.loadScriptGuidance = [
'This dataset comes from another Qlik app',
'Use Binary load to get the entire data model',
'Or use specific LOAD statements from the source app'
];
// Try to find the source app
if (dataset.connectionId) {
try {
const sourceApp = await this.findSourceApp(dataset.connectionId);
if (sourceApp) {
connectionInfo.sourceApp = sourceApp;
connectionInfo.binaryLoadSyntax = `Binary [lib://${dataset.spaceName}/${sourceApp.name}];`;
}
} catch (error) {
console.error('[DataCatalogService] Could not find source app:', error);
}
}
} else if (dataset.isConnectionBased) {
// External data connection (database, etc.)
connectionInfo.sourceType = 'database';
connectionInfo.connectionType = dataset.connectionType;
connectionInfo.connectionName = dataset.connectionName;
// Build load script template
connectionInfo.connectionTemplate = `LIB CONNECT TO '${dataset.spaceName}:${dataset.connectionName}';`;
// Add the technical name if available
if (dataset.technicalName) {
connectionInfo.loadTemplate = `
LIB CONNECT TO '${dataset.spaceName}:${dataset.connectionName}';
[${dataset.itemName}]:
LOAD *;
SQL SELECT * FROM ${dataset.technicalName};
`;
} else if (dataset.projectId && dataset.datasetName && dataset.tableName) {
// BigQuery specific
connectionInfo.loadTemplate = `
LIB CONNECT TO '${dataset.connectionName}';
[${dataset.itemName}]:
LOAD *;
SQL SELECT * FROM \`${dataset.projectId}.${dataset.datasetName}.${dataset.tableName}\`;
`;
}
// Include field information for selective loading
if (dataset.fields && dataset.fields.length > 0) {
const fieldList = dataset.fields.map((f: any) => f.name || f.datasetFieldName).join(',\n ');
connectionInfo.selectiveLoadTemplate = `
LIB CONNECT TO '${dataset.connectionName}';
[${dataset.itemName}]:
LOAD
${fieldList};
SQL SELECT
${fieldList}
FROM ${dataset.technicalName || dataset.tableName};
`;
}
} else {
// File-based dataset
const fileName = dataset.itemName;
const spaceName = dataset.spaceName;
// Handle missing space name
if (!fileName) {
connectionInfo.error = 'No file name available';
return connectionInfo;
}
// If we don't have space name, warn about it
let effectiveSpaceName = spaceName;
if (!spaceName && dataset.spaceId) {
connectionInfo.warning = `Space name not resolved. Using placeholder for space ID: ${dataset.spaceId}`;
connectionInfo.spacePlaceholder = `<SPACE_NAME_FOR_${dataset.spaceId}>`;
effectiveSpaceName = connectionInfo.spacePlaceholder;
} else if (!spaceName) {
effectiveSpaceName = 'DataFiles';
}
// Set basic file info
connectionInfo.fileName = fileName;
connectionInfo.spaceName = effectiveSpaceName;
connectionInfo.datasetType = dataset.resourceType || dataset.type || dataset.itemType;
connectionInfo.sourceType = 'file';
connectionInfo.fileExtension = fileName.includes('.') ? fileName.split('.').pop()?.toLowerCase() : null;
// Generate file connection path - fileName should already include the extension
connectionInfo.connectionTemplate = `lib://${effectiveSpaceName}:DataFiles/${fileName}`;
// Generate load syntax based on file type
let loadSyntax = '';
switch (connectionInfo.fileExtension) {
case 'qvd':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (qvd);`;
break;
case 'csv':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (txt, utf8, embedded labels, delimiter is ',');`;
break;
case 'xlsx':
case 'xls':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (ooxml, embedded labels, table is Sheet1);`;
break;
case 'parquet':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (parquet);`;
break;
case 'txt':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (txt, utf8, embedded labels, delimiter is '\\t');`;
break;
case 'json':
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}] (json);`;
break;
default:
loadSyntax = `LOAD * FROM [${connectionInfo.connectionTemplate}];`;
}
connectionInfo.loadSyntax = loadSyntax;
// Let Claude determine the best load approach based on the actual data
connectionInfo.datasetMetadata = {
resourceId: dataset.resourceId || dataset.datasetId,
itemId: dataset.itemId || dataset.id,
rowCount: dataset.rowCount,
columnCount: dataset.columnCount || dataset.fieldCount,
fields: dataset.fields || dataset.columns || [],
size: dataset.size,
createdDate: dataset.createdDate,
modifiedDate: dataset.modifiedDate,
sourceInfo: {
type: 'file',
connectionId: dataset.connectionId,
sourcePath: dataset.sourcePath
}
};
// Add notes about what Claude should consider
connectionInfo.loadScriptGuidance = [
'This appears to be a file-based dataset',
'Use the lib://Space:DataFiles/filename syntax',
`File type: ${connectionInfo.fileExtension || 'unknown'}`,
`Space: ${effectiveSpaceName}`
];
if (connectionInfo.warning) {
connectionInfo.loadScriptGuidance.unshift(connectionInfo.warning);
}
}
// Include any schema information if available
if (dataset.schema) {
connectionInfo.schema = dataset.schema;
}
// Include data quality indicators if available
if (dataset.dataQuality) {
connectionInfo.dataQuality = dataset.dataQuality;
}
return connectionInfo;
}
/**
* Batch get dataset details with space names
*/
async getMultipleDatasetDetails(itemIds: string[]): Promise<any[]> {
logger.debug(`[DataCatalogService] Getting details for ${itemIds.length} datasets`);
const results = await Promise.all(
itemIds.map(async (itemId) => {
try {
return await this.getDatasetDetails(itemId);
} catch (error) {
console.error(`[DataCatalogService] Failed to get details for ${itemId}:`, error);
return {
itemId,
error: error instanceof Error ? error.message : 'Unknown error'
};
}
})
);
return results;
}
/**
* ENHANCED: Search for data assets across all spaces
*/
async searchDataAssets(query: string, options: { limit?: number; offset?: number } = {}): Promise<any> {
const { limit = 50, offset = 0 } = options;
try {
// Check cache first
const cacheKey = `data-assets:${query}:${limit}:${offset}`;
const cachedResult = this.dataAssetSearchCache.get(cacheKey);
if (cachedResult) {
return cachedResult;
}
// Get all spaces with their items
const spaces = await this.getSpacesCatalog({
includeItems: true,
includeMembers: false
});
// Collect all data assets
const allDataAssets: any[] = [];
for (const space of spaces.spaces) {
const spaceDataAssets = space.dataAssets.map(asset => ({
...asset,
spaceName: space.name,
spaceId: space.id,
spaceType: space.type,
spaceOwner: space.spaceInfo.ownerName
}));
allDataAssets.push(...spaceDataAssets);
}
// Filter by query
const queryLower = query.toLowerCase();
const filteredAssets = allDataAssets.filter(asset =>
asset.name.toLowerCase().includes(queryLower) ||
asset.description?.toLowerCase().includes(queryLower) ||
asset.spaceName.toLowerCase().includes(queryLower) ||
asset.tags?.some((tag: string) => tag.toLowerCase().includes(queryLower))
);
// Apply pagination
const paginatedAssets = filteredAssets.slice(offset, offset + limit);
const result = {
data: paginatedAssets,
meta: {
total: filteredAssets.length,
offset,
limit,
hasMore: filteredAssets.length > offset + limit
}
};
// Cache the result (store just the result object)
this.dataAssetSearchCache.set(cacheKey, result);
return result;
} catch (error) {
console.error('[DataCatalogService] Failed to search data assets:', error);
throw error;
}
}
/**
* Build catalog result with consistent structure
*/
private buildCatalogResult(
spaces: QlikSpace[],
totalBeforePagination: number,
searchTime: number
): SpaceCatalogResult {
const facets = this.buildFacets(spaces);
const summary = this.buildSummary(spaces);
return {
spaces,
totalCount: totalBeforePagination || spaces.length,
searchTime,
facets,
summary
};
}
/**
* Generate cache key for options
*/
private generateCacheKey(options: any): string {
const keyParts = [
'spaces',
options.query || '',
options.spaceType || 'all',
options.ownerId || '',
options.memberUserId || '',
options.hasDataAssets !== undefined ? `hasData:${options.hasDataAssets}` : '',
options.minItems || '',
options.sortBy || '',
options.sortOrder || '',
options.includeMembers ? 'members' : '',
options.includeItems ? 'items' : '',
options.getAllSpaces ? 'getAllSpaces' : '',
options.effectiveLimit || options.limit || '100',
options.offset || '0'
];
return keyParts.filter(Boolean).join(':');
}
/**
* Update performance metrics
*/
private updatePerformanceMetrics(searchTime: number): void {
this.searchPerformance.averageSearchTime =
(this.searchPerformance.averageSearchTime * (this.searchPerformance.totalSearches - 1) + searchTime) /
this.searchPerformance.totalSearches;
}
/**
* ENHANCED: Build comprehensive content index for better search
*/
private async buildContentSearchIndex(spaces: QlikSpace[]): Promise<void> {
logger.debug('[SpacesCatalog] Building enhanced search index...');
this.contentSearchIndex.clear();
for (const space of spaces) {
// Index space-level content
this.indexContent(space.name.toLowerCase(), space.id);
this.indexContent(space.description?.toLowerCase() || '', space.id);
this.indexContent(space.spaceInfo.ownerName.toLowerCase(), space.id);
// Index all items in the space
const allItems = [...space.dataAssets, ...space.spaceItems];
for (const item of allItems) {
this.indexContent(item.name.toLowerCase(), space.id);
this.indexContent(item.description?.toLowerCase() || '', space.id);
// Index tags
if (item.tags) {
for (const tag of item.tags) {
this.indexContent(tag.toLowerCase(), space.id);
}
}
}
// Index member names
for (const member of space.members) {
this.indexContent(member.userName.toLowerCase(), space.id);
}
}
logger.debug(`[SpacesCatalog] Indexed ${this.contentSearchIndex.size} unique terms`);
}
/**
* Index content for search
*/
private indexContent(content: string, spaceId: string): void {
if (!content) return;
// Tokenize content
const tokens = content.split(/[\s,._\-\/]+/).filter(t => t.length > 0);
for (const token of tokens) {
if (!this.contentSearchIndex.has(token)) {
this.contentSearchIndex.set(token, new Set<string>());
}
this.contentSearchIndex.get(token)!.add(spaceId);
}
}
/**
* Enhanced search using content index
*/
private searchUsingIndex(query: string): Set<string> {
const queryTokens = query.toLowerCase().split(/[\s,._\-\/]+/).filter(t => t.length > 0);
const matchingSpaceIds = new Set<string>();
for (const token of queryTokens) {
// Exact match
if (this.contentSearchIndex.has(token)) {
for (const spaceId of this.contentSearchIndex.get(token)!) {
matchingSpaceIds.add(spaceId);
}
}
// Prefix match
for (const [indexedToken, spaceIds] of this.contentSearchIndex.entries()) {
if (indexedToken.startsWith(token)) {
for (const spaceId of spaceIds) {
matchingSpaceIds.add(spaceId);
}
}
}
}
return matchingSpaceIds;
}
/**
* Group by helper
*/
private groupBy<T>(array: T[], key: keyof T): Record<string, number> {
return array.reduce((result: Record<string, number>, item) => {
const value = String(item[key]);
result[value] = (result[value] || 0) + 1;
return result;
}, {});
}
/**
* Fetch space members
*/
/**
* Fetch space members with resolved names
*/
/**
* Fetch space members with resolved names
*/
private async fetchSpaceMembers(spaceId: string): Promise<any[]> {
try {
const response = await this.apiClient.makeRequest(`/api/v1/spaces/${spaceId}/assignments`, 'GET');
const members = response.data || response || [];
// THE FIX: Use assigneeId, not userId!
const userIds = members
.map((member: any) => member.assigneeId) // <-- THIS IS THE KEY CHANGE
.filter(Boolean);
if (userIds.length === 0) return members;
// Resolve all user IDs to get names
logger.debug(`[SpacesCatalog] Resolving ${userIds.length} member names for space ${spaceId}`);
const resolvedUsers = await this.apiClient.resolveOwnersToUsers(userIds);
// Enhance members with resolved names
const enhancedMembers = members.map((member: any) => {
const actualUserId = member.assigneeId; // <-- USE assigneeId!
const resolvedUser = resolvedUsers.get(actualUserId);
return {
...member,
userId: actualUserId, // Store the actual user ID
userName: resolvedUser?.displayName || `User ${actualUserId?.substring(0, 8)}...`,
userEmail: resolvedUser?.email || 'unknown@example.com',
role: member.roles?.[0] || 'member', // roles is an array, get first
status: resolvedUser?.status || 'unknown'
};
});
logger.debug(`[SpacesCatalog] Resolved ${enhancedMembers.length} members with names`);
return enhancedMembers;
} catch (error) {
console.error(`[SpacesCatalog] Failed to fetch members for space ${spaceId}:`, error);
return [];
}
}
/**
* Fetch space items
*/
private async fetchSpaceItems(spaceId: string): Promise<{ dataAssets: any[], spaceItems: any[] }> {
try {
// Add includeResourceAttributes to get resourceSubType
const url = `/items?spaceId=${spaceId}&limit=100&offset=0&includeResourceAttributes=true`;
const response = await this.apiClient.makeRequest(url, 'GET');
const items = response.data || response || [];
// Separate data assets from other items
const dataAssets: any[] = [];
const spaceItems: any[] = [];
items.forEach((item: any) => {
// Extract resourceSubType from resourceAttributes if available
const resourceSubType = item.resourceAttributes?.resourceSubType || item.resourceSubType;
const normalizedType = this.normalizeItemType(
item.resourceType || item.type,
item.name,
item.resourceSubType // Now we have the correct subtype
);
// Store the resourceSubType on the item for later use
item.resourceSubType = resourceSubType;
if (normalizedType === 'dataset') {
dataAssets.push(item);
} else {
spaceItems.push(item);
}
});
return { dataAssets, spaceItems };
} catch (error) {
console.error(`[SpacesCatalog] Failed to fetch items for space ${spaceId}:`, error);
return { dataAssets: [], spaceItems: [] };
}
}
/**
* Find source app for QIX datasets
*/
private async findSourceApp(connectionId: string): Promise<any> {
try {
// Try to find the app directly
const searchResult = await this.apiClient.makeRequest('/api/v1/apps', 'GET');
const apps = searchResult.data || searchResult || [];
// Try to match by ID or other criteria
const sourceApp = apps.find((app: any) =>
app.id === connectionId ||
app.resourceId === connectionId
);
if (sourceApp) {
return {
id: sourceApp.id,
name: sourceApp.name,
spaceId: sourceApp.spaceId,
spaceName: sourceApp.space
};
}
} catch (error) {
console.error('[DataCatalogService] Error finding source app:', error);
}
return null;
}
/**
* Clear all caches
*/
clearCaches(): void {
this.spacesCache.clear();
this.spaceDetailsCache.clear();
this.dataAssetSearchCache.clear();
this.contentSearchIndex.clear();
logger.debug('[DataCatalogService] All caches cleared');
}
/**
* Get cache statistics
*/
getCacheStats(): any {
return {
spacesCache: this.spacesCache.size,
spaceDetailsCache: this.spaceDetailsCache.size,
dataAssetSearchCache: this.dataAssetSearchCache.size,
contentSearchIndex: this.contentSearchIndex.size,
performance: this.searchPerformance
};
}
/**
* Get data connections for a space
*/
async getSpaceDataConnections(spaceId: string): Promise<any[]> {
try {
const response = await this.apiClient.makeRequest(`/api/v1/data-connections?spaceId=${spaceId}`, 'GET');
return response.data || response || [];
} catch (error) {
console.error(`[DataCatalogService] Failed to get data connections for space ${spaceId}:`, error);
return [];
}
}
/**
* Get comprehensive content statistics with detailed space and asset type breakdown
*/
async getContentStatistics(options: {
groupBy?: 'space' | 'type' | 'owner';
// Filter parameters (keep these)
spaceName?: string;
spaceId?: string;
ownerName?: string;
ownerId?: string;
// Options (keep these)
includeSpaceBreakdown?: boolean;
includeAppSubtypes?: boolean;
includeDetailedTable?: boolean;
includeSizeInfo?: boolean;
includeDateMetrics?: boolean;
includeUserMetrics?: boolean;
fileSizeSampleLimit?: number;
} = {}): Promise<ContentStatistics> {
const startTime = Date.now();
console.error('[DataCatalogService] Getting comprehensive content statistics with filtering...');
try {
// Step 1: Determine target spaces based on filters
let targetSpaces: QlikSpace[] = [];
if (options.spaceId || options.spaceName || options.ownerId || options.ownerName) {
// We have filters - get filtered spaces
console.error('[ContentStats] Applying space filters...');
// Get minimal space info first (without items)
const spaceCatalog = await this.getSpacesCatalog({
query: options.spaceName,
includeMembers: true,
includeItems: false // Don't fetch items here
});
// Apply filters
targetSpaces = spaceCatalog.spaces.filter((space: QlikSpace) => {
if (options.spaceId && space.id !== options.spaceId) return false;
if (options.spaceName) {
const searchName = options.spaceName.toLowerCase();
if (!space.name.toLowerCase().includes(searchName)) return false;
}
if (options.ownerId && space.spaceInfo?.ownerId !== options.ownerId) return false;
if (options.ownerName) {
const searchOwner = options.ownerName.toLowerCase();
const ownerName = (space.spaceInfo?.ownerName || '').toLowerCase();
if (!ownerName.includes(searchOwner)) return false;
}
return true;
});
console.error(`[ContentStats] Filtered to ${targetSpaces.length} spaces`);
} else {
// No filters - get all spaces
console.error('[ContentStats] No filters - getting all spaces');
const spaceCatalog = await this.getSpacesCatalog({
includeMembers: true,
includeItems: false // Don't fetch items here
});
targetSpaces = spaceCatalog.spaces;
}
if (targetSpaces.length === 0) {
console.error('[ContentStats] No spaces match the filters');
return this.createEmptyStatistics();
}
// Step 2: Fetch ALL items from target spaces
console.error(`[ContentStats] Fetching items from ${targetSpaces.length} space(s)...`);
let allItems: any[] = [];
if (options.spaceId) {
// Single space - direct fetch
const itemsResult = await this.getSpaceItems({
spaceId: options.spaceId,
includeSpaceInfo: true,
skipPagination: true,
limit: 10000 // Get all items
});
allItems = itemsResult.items || [];
} else if (targetSpaces.length === 1) {
// Single filtered space
const itemsResult = await this.getSpaceItems({
spaceId: targetSpaces[0].id,
includeSpaceInfo: true,
skipPagination: true,
limit: 10000
});
allItems = itemsResult.items || [];
} else {
// Multiple spaces - fetch from all
const itemsResult = await this.getSpaceItems({
spaceIds: targetSpaces.map(s => s.id),
includeSpaceInfo: true,
limit: 10000
});
allItems = itemsResult.items || [];
}
console.error(`[ContentStats] Fetched ${allItems.length} total items`);
// Step 3: Process items and prepare for size enrichment
console.error('[ContentStats] Processing items and normalizing types...');
// Initialize accumulators
const typeCountMap = new Map<string, number>();
const spaceItemsMap = new Map<string, any[]>();
const ownerItemsMap = new Map<string, any[]>();
const datasetsForSizeEnrichment: any[] = [];
let totalItems = 0;
let totalDataAssets = 0;
// Process each item
for (const item of allItems) {
// Normalize the type
const normalizedType = this.normalizeItemType(
item.resourceType || item.type,
item.name,
item.resourceSubType,
item.resourceAttributes
);
// Store normalized type on the item for later use
item._normalizedType = normalizedType;
// Count by type
typeCountMap.set(normalizedType, (typeCountMap.get(normalizedType) || 0) + 1);
totalItems++;
// Track datasets for size enrichment
if (normalizedType === 'dataset' || this.isDataFile(item)) {
datasetsForSizeEnrichment.push(item);
totalDataAssets++;
}
// Group by space
const spaceName = item.space?.name || item._spaceName || 'Unknown';
if (!spaceItemsMap.has(spaceName)) {
spaceItemsMap.set(spaceName, []);
}
spaceItemsMap.get(spaceName)!.push(item);
// Group by owner (if needed)
if (options.includeUserMetrics !== false || options.groupBy === 'owner') {
const ownerId = item.ownerId || item.owner?.id || 'unknown';
if (!ownerItemsMap.has(ownerId)) {
ownerItemsMap.set(ownerId, []);
}
ownerItemsMap.get(ownerId)!.push(item);
}
}
console.error(`[ContentStats] Normalized types: ${typeCountMap.size} unique types`);
console.error(`[ContentStats] Found ${datasetsForSizeEnrichment.length} datasets for size enrichment`);
// Step 4: Enrich datasets with size information
let totalStorage = 0;
const typeSizeMap = new Map<string, number>();
const spaceSizeMap = new Map<string, number>();
if (options.includeSizeInfo !== false && datasetsForSizeEnrichment.length > 0) {
console.error(`[ContentStats] Getting size info for ${datasetsForSizeEnrichment.length} datasets...`);
const sizeStats = await this.enrichDataFilesWithSize(datasetsForSizeEnrichment);
// Update items with size info and accumulate
if (sizeStats && sizeStats.enrichedItems) {
for (const enrichedItem of sizeStats.enrichedItems) {
const size = enrichedItem.size || 0;
if (size > 0) {
totalStorage += size;
// Accumulate by type
const type = enrichedItem._normalizedType || 'dataset';
typeSizeMap.set(type, (typeSizeMap.get(type) || 0) + size);
// Accumulate by space
const spaceName = enrichedItem.space?.name || enrichedItem._spaceName || 'Unknown';
spaceSizeMap.set(spaceName, (spaceSizeMap.get(spaceName) || 0) + size);
}
}
}
console.error(`[ContentStats] Total storage: ${this.formatFileSize(totalStorage)}`);
}
// Step 5: Build type distribution
const typeDistribution: Record<string, any> = {};
const assetTypes = Array.from(typeCountMap.keys()).sort();
for (const [type, count] of typeCountMap.entries()) {
typeDistribution[type] = {
count: count,
percentage: totalItems > 0 ? Math.round((count / totalItems) * 100) : 0,
spacesWithType: 0, // Will calculate below
uniqueOwners: 0, // Will calculate below
totalSizeFormatted: undefined,
averageSize: undefined
};
// Add size info if available
const typeSize = typeSizeMap.get(type) || 0;
if (typeSize > 0) {
typeDistribution[type].totalSizeFormatted = this.formatFileSize(typeSize);
typeDistribution[type].averageSize = this.formatFileSize(Math.round(typeSize / count));
}
}
// Calculate spaces with type and unique owners
for (const [spaceName, items] of spaceItemsMap.entries()) {
const typesInSpace = new Set<string>();
const ownersByType = new Map<string, Set<string>>();
for (const item of items) {
const type = item._normalizedType;
typesInSpace.add(type);
if (!ownersByType.has(type)) {
ownersByType.set(type, new Set());
}
const ownerId = item.ownerId || item.owner?.id || 'unknown';
ownersByType.get(type)!.add(ownerId);
}
// Update spaces with type count
for (const type of typesInSpace) {
if (typeDistribution[type]) {
typeDistribution[type].spacesWithType++;
}
}
// Update unique owners count
for (const [type, owners] of ownersByType.entries()) {
if (typeDistribution[type]) {
typeDistribution[type].uniqueOwners = owners.size;
}
}
}// Step 6: Build detailed table (Space × Asset Type matrix)
const detailedTable: any[][] = [];
if (options.includeDetailedTable !== false) {
for (const space of targetSpaces) {
const spaceItems = spaceItemsMap.get(space.name) || [];
const spaceTypeCount = new Map<string, number>();
// Count items by type for this space
for (const item of spaceItems) {
const type = item._normalizedType;
spaceTypeCount.set(type, (spaceTypeCount.get(type) || 0) + 1);
}
// Build row
const row: any[] = [
space.name,
space.type || 'unknown',
space.spaceInfo?.ownerName || 'Unknown',
spaceItems.length // Total items
];
// Add count for each asset type
for (const assetType of assetTypes) {
row.push(spaceTypeCount.get(assetType) || 0);
}
// Add members count
row.push(space.statistics?.totalMembers || 0);
// Add storage in MB
const spaceStorage = spaceSizeMap.get(space.name) || 0;
row.push(spaceStorage > 0 ? Math.round(spaceStorage / (1024 * 1024)) : 0);
detailedTable.push(row);
}
// Sort by total items (descending)
detailedTable.sort((a, b) => b[3] - a[3]);
}
// Step 7: Build final statistics
const totalMembers = targetSpaces.reduce((sum, space) =>
sum + (space.statistics?.totalMembers || 0), 0);
const statistics: ContentStatistics = {
summary: {
totalSpaces: targetSpaces.length,
totalItems,
totalDataAssets,
totalMembers,
totalStorage,
averageItemsPerSpace: targetSpaces.length > 0 ?
Math.round(totalItems / targetSpaces.length) : 0,
averageMembersPerSpace: targetSpaces.length > 0 ?
Math.round(totalMembers / targetSpaces.length) : 0
},
assetTypes,
typeDistribution,
detailedTable,
spaceBreakdown: {
personal: targetSpaces.filter(s => s.type === 'personal').length,
shared: targetSpaces.filter(s => s.type === 'shared').length,
managed: targetSpaces.filter(s => s.type === 'managed').length,
data: targetSpaces.filter(s => s.type === 'data').length
},
lastUpdated: new Date().toISOString()
};
// Add optional sections based on options...
// (dateMetrics, userMetrics, groupBy views - keep existing logic)
const executionTime = Date.now() - startTime;
console.error(`[ContentStats] Statistics generated in ${executionTime}ms`);
return statistics;
} catch (error) {
console.error('[DataCatalogService] Failed to get content statistics:', error);
throw error;
}
}
private createEmptyStatistics(): ContentStatistics {
return {
summary: {
totalSpaces: 0,
totalItems: 0,
totalDataAssets: 0,
totalMembers: 0,
totalStorage: 0,
averageItemsPerSpace: 0,
averageMembersPerSpace: 0
},
assetTypes: [],
typeDistribution: {},
detailedTable: [],
spaceBreakdown: {
personal: 0,
shared: 0,
managed: 0,
data: 0
},
lastUpdated: new Date().toISOString(),
// Optional sections
dateMetrics: {
oldestContent: null,
newestContent: null,
contentByMonth: [],
monthsWithContent: 0
},
userMetrics: {
totalContentOwners: 0,
topContentOwners: [],
averageItemsPerOwner: 0
}
};
}
private isDataFile(item: any): boolean {
const name = (item.name || '').toLowerCase();
const resourceType = (item.resourceType || '').toLowerCase();
const dataExtensions = [
'.qvd', '.csv', '.xlsx', '.xls', '.parquet',
'.json', '.xml', '.txt', '.tsv', '.pdf'
];
return dataExtensions.some(ext => name.endsWith(ext)) ||
resourceType === 'dataset' ||
resourceType === 'datasource';
}
/**
* Helper method to get dataset size from multiple sources - FIXED VERSION
*/
private async getDatasetSize(item: any): Promise<number> {
const resourceId = item.resourceId;
const itemId = item.id;
logger.debug(`[DEBUG] Getting size for: ${item.name}`);
logger.debug(`[DEBUG] ResourceId: ${resourceId}, ItemId: ${itemId}`);
// Try /data-sets endpoint first
if (resourceId) {
try {
logger.debug(`[DEBUG] Calling /data-sets/${resourceId}`);
const response = await this.apiClient.makeRequest(`/api/v1/data-sets/${resourceId}`, 'GET');
// LOG EVERYTHING
logger.debug(`[DEBUG] Full Response Type: ${typeof response}`);
logger.debug(`[DEBUG] Response Keys: ${response ? Object.keys(response).join(', ') : 'null'}`);
// Check if response has data wrapper
if (response?.data) {
logger.debug(`[DEBUG] Response has .data wrapper`);
logger.debug(`[DEBUG] response.data keys: ${Object.keys(response.data).join(', ')}`);
}
// Check operational field
if (response?.operational) {
logger.debug(`[DEBUG] response.operational exists: ${JSON.stringify(response.operational)}`);
}
if (response?.data?.operational) {
logger.debug(`[DEBUG] response.data.operational exists: ${JSON.stringify(response.data.operational)}`);
}
// Try all possible paths
const possibleSizes = [
{ path: 'response.operational.size', value: response?.operational?.size },
{ path: 'response.data.operational.size', value: response?.data?.operational?.size },
{ path: 'response.size', value: response?.size },
{ path: 'response.data.size', value: response?.data?.size },
];
logger.debug(`[DEBUG] Checking all possible size locations:`);
possibleSizes.forEach(p => {
logger.debug(` ${p.path}: ${p.value || 'undefined'}`);
});
// Get size from wherever it exists
let size = 0;
for (const possible of possibleSizes) {
if (possible.value && possible.value > 0) {
size = possible.value;
logger.debug(`[DEBUG] ✓ Found size at ${possible.path}: ${size}`);
break;
}
}
// Also check row count
const rowCount = response?.operational?.rowCount ||
response?.data?.operational?.rowCount ||
response?.rowCount ||
response?.data?.rowCount;
if (rowCount) {
logger.debug(`[DEBUG] Found rowCount: ${rowCount}`);
item.rowCount = rowCount;
}
if (size > 0) {
return size;
}
logger.debug(`[DEBUG] No size found in /data-sets response`);
} catch (error: any) {
logger.debug(`[DEBUG] /data-sets API error: ${error.message}`);
logger.debug(`[DEBUG] Error status: ${error.status || 'unknown'}`);
}
} else {
logger.debug(`[DEBUG] No resourceId available for ${item.name}`);
}
// Try /items endpoint as fallback
if (itemId) {
try {
logger.debug(`[DEBUG] Trying /api/v1/items/${itemId}`);
const response = await this.apiClient.makeRequest(`/api/v1/items/${itemId}`, 'GET');
logger.debug(`[DEBUG] /items response type: ${typeof response}`);
logger.debug(`[DEBUG] /items keys: ${response ? Object.keys(response).slice(0, 10).join(', ') : 'null'}`);
// Check for resourceId if we don't have it
if (!resourceId && response?.resourceId) {
logger.debug(`[DEBUG] Found resourceId in /items: ${response.resourceId}`);
item.resourceId = response.resourceId;
// Now try /data-sets again with the new resourceId
return await this.getDatasetSize(item);
}
const size = response?.size || response?.data?.size || response?.resourceSize || 0;
if (size > 0) {
logger.debug(`[DEBUG] Found size in /items: ${size}`);
return size;
}
} catch (error: any) {
logger.debug(`[DEBUG] /items API error: ${error.message}`);
}
}
logger.debug(`[DEBUG] ✗ No size found for ${item.name}`);
return 0;
}
/**
* Get lightweight counts for a space without fetching actual items
*/
private async getSpaceCounts(spaceId: string): Promise<{
itemCount: number;
memberCount: number;
}> {
try {
logger.debug(`[SpacesCatalog] Getting counts for space ${spaceId}`);
const seenItemIds = new Set<string>();
let currentUrl: string | null = `/items?spaceId=${spaceId}&limit=100&offset=0`;
let pageNum = 1;
while (currentUrl) {
const itemsResponse = await this.apiClient.makeRequest(currentUrl, 'GET');
const items = itemsResponse?.data || [];
logger.debug(`[SpacesCatalog] Page ${pageNum}: Got ${items.length} items`);
// Unique item'ları say
let newItems = 0;
for (const item of items) {
if (item.id && !seenItemIds.has(item.id)) {
seenItemIds.add(item.id);
newItems++;
}
}
logger.debug(`[SpacesCatalog] Page ${pageNum}: ${newItems} new, ${seenItemIds.size} total unique`);
// Next URL kontrolü
if (itemsResponse?.links?.next?.href) {
let nextHref = itemsResponse.links.next.href;
// URL temizleme - /api/v1 kaldır
nextHref = nextHref.replace('/api/v1/', '/');
// Full URL ise path kısmını al
if (nextHref.startsWith('http')) {
const url = new URL(nextHref);
currentUrl = url.pathname.replace('/api/v1/', '/') + url.search;
} else {
currentUrl = nextHref;
}
logger.debug(`[SpacesCatalog] Next URL: ${currentUrl}`);
} else {
logger.debug(`[SpacesCatalog] No next link, done`);
currentUrl = null;
}
pageNum++;
// Güvenlik
if (pageNum > 50) {
logger.debug(`[SpacesCatalog] Max pages reached`);
break;
}
}
// Members
const membersResponse = await this.apiClient.makeRequest(
`/api/v1/spaces/${spaceId}/assignments?limit=100`,
'GET'
);
const memberCount = (membersResponse?.data || []).length;
const totalItemCount = seenItemIds.size;
logger.debug(`[SpacesCatalog] ✅ FINAL: ${totalItemCount} unique items, ${memberCount} members`);
return { itemCount: totalItemCount, memberCount };
} catch (error) {
logger.debug(`[SpacesCatalog] Failed: ${error instanceof Error ? error.message : String(error)}`);
return { itemCount: 0, memberCount: 0 };
}
}
/**
* Enrich data files with size - DEBUG VERSION
*/
private async enrichDataFilesWithSize(items: any[], sampleLimit?: number): Promise<any> {
const startTime = Date.now();
logger.debug('[DataCatalogService] === Starting Size Enrichment ===');
logger.debug(`[DataCatalogService] Total items: ${items.length}`);
// Filter for actual datasets
const dataFiles = items.filter(item => {
const resourceType = (item.resourceType || '').toLowerCase();
const normalizedType = (item.normalizeItemType || '').toLowerCase();
const itemName = (item.name || '').toLowerCase();
// Skip non-dataset types
const skipTypes = ['app', 'assistant', 'automation', 'note', 'link', 'genericlink'];
if (skipTypes.includes(resourceType)) {
return false;
}
// Include if it's a dataset
const isDataset = (
resourceType === 'dataset' ||
normalizedType === 'dataset' ||
itemName.endsWith('.qvd') ||
itemName.endsWith('.csv') ||
itemName.endsWith('.xlsx') ||
itemName.endsWith('.xls') ||
itemName.endsWith('.parquet') ||
itemName.endsWith('.txt') ||
itemName.endsWith('.json') ||
itemName.endsWith('.pdf')
);
return isDataset;
});
logger.debug(`[DataCatalogService] Found ${dataFiles.length} datasets`);
// IMPORTANT: Process ALL files, not just first 3!
const filesToProcess = sampleLimit && sampleLimit < dataFiles.length ?
dataFiles.slice(0, sampleLimit) :
dataFiles; // Process ALL if no limit or limit > count
logger.debug(`[DataCatalogService] Will process ${filesToProcess.length} datasets`);
// Tracking variables
let totalSizeBytes = 0;
let successCount = 0;
let failureCount = 0;
const sizeBySpace: Record<string, number> = {};
const countBySpace: Record<string, number> = {};
const results: any[] = [];
// Process in batches
const batchSize = 5;
for (let i = 0; i < filesToProcess.length; i += batchSize) {
const batch = filesToProcess.slice(i, i + batchSize);
const batchNum = Math.floor(i / batchSize) + 1;
const totalBatches = Math.ceil(filesToProcess.length / batchSize);
logger.debug(`[DataCatalogService] Processing batch ${batchNum}/${totalBatches} (items ${i + 1}-${Math.min(i + batchSize, filesToProcess.length)})`);
const batchPromises = batch.map(async (item) => {
try {
// Get resourceId if not present
if (!item.resourceId && item.id) {
try {
const itemResponse = await this.apiClient.makeRequest(`/api/v1/items/${item.id}`, 'GET');
const itemData = itemResponse?.data || itemResponse;
item.resourceId = itemData.resourceId;
} catch (e) {
logger.debug(`[DataCatalogService] Failed to get resourceId for ${item.name}`);
}
}
// Get size using getDatasetSize method
const size = await this.getDatasetSize(item);
if (size > 0) {
item.size = size;
item.sizeFormatted = this.formatFileSize(size);
// Track totals
totalSizeBytes += size;
successCount++;
// Track by space
const spaceName = item.space?.name || item.spaceName || 'Unknown';
if (!sizeBySpace[spaceName]) {
sizeBySpace[spaceName] = 0;
countBySpace[spaceName] = 0;
}
sizeBySpace[spaceName] += size;
countBySpace[spaceName]++;
// Add to results
results.push({
name: item.name,
size: size,
sizeFormatted: this.formatFileSize(size)
});
logger.debug(`[DataCatalogService] ✓ ${item.name}: ${this.formatFileSize(size)}`);
return { success: true, item };
} else {
failureCount++;
logger.debug(`[DataCatalogService] ✗ No size for ${item.name}`);
return { success: false, item };
}
} catch (error: any) {
failureCount++;
console.error(`[DataCatalogService] Error processing ${item.name}: ${error.message}`);
return { success: false, item };
}
});
// Wait for batch to complete
await Promise.all(batchPromises);
}
const processingTime = Date.now() - startTime;
// Build summary
const summary = {
processed: filesToProcess.length,
successful: successCount,
failed: failureCount,
totalSizeBytes: totalSizeBytes,
totalSizeFormatted: this.formatFileSize(totalSizeBytes),
sizeBySpace: Object.entries(sizeBySpace).map(([space, bytes]) => ({
space: space,
sizeBytes: bytes,
sizeFormatted: this.formatFileSize(bytes),
fileCount: countBySpace[space]
})),
results: results,
processingTimeMs: processingTime
};
logger.debug('[DataCatalogService] === Size Enrichment Complete ===');
logger.debug(`[DataCatalogService] Processed: ${summary.processed}, Success: ${summary.successful}, Failed: ${summary.failed}`);
logger.debug(`[DataCatalogService] Total Size: ${summary.totalSizeFormatted}`);
return summary;
}
/**
* Get detailed information about a specific space
*/
async getSpaceDetails(
spaceId: string,
includeMembers: boolean = true,
includeItems: boolean = true
): Promise<QlikSpace | null> {
try {
logger.debug(`[SpacesCatalog] Getting details for space: ${spaceId}`);
// Check cache first
const cacheKey = spaceId;
if (this.spaceDetailsCache.has(cacheKey)) {
return this.spaceDetailsCache.get(cacheKey)!;
}
// Fetch space basic info
const spaceInfo = await this.apiClient.makeRequest(`/api/v1/spaces/${spaceId}`, 'GET');
// Resolve owner
const ownerId = spaceInfo.ownerId || spaceInfo.owner?.id;
const resolvedUsers = ownerId ?
await this.apiClient.resolveOwnersToUsers([ownerId]) :
new Map<string, ResolvedUser>();
// Create full space
const detailedSpace = await this.createFullSpace(
spaceInfo,
includeMembers,
includeItems
);
// Cache the result
this.spaceDetailsCache.set(cacheKey, detailedSpace);
return detailedSpace;
} catch (error) {
console.error(`[SpacesCatalog] Failed to get space details for ${spaceId}:`, error);
return null;
}
}
/**
* Unified method to get items from spaces with flexible filtering
*/
async getSpaceItems(options: SpaceItemsOptions = {}): Promise<SpaceItemsResult> {
const startTime = Date.now();
try {
logger.debug(`[DataCatalogService] Getting space items with options: ${JSON.stringify(options)}`);
// Handle date filtering
let effectiveStartDate: string | null = null;
let effectiveEndDate: string | null = null;
if (options.startDate || options.endDate) {
const now = new Date();
const startDate = options.startDate ? new Date(options.startDate) : new Date(0);
const endDate = options.endDate ? new Date(options.endDate) : now;
if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) {
throw new Error('Invalid date format. Use ISO format (YYYY-MM-DD or YYYY-MM-DDTHH:mm:ss)');
}
effectiveStartDate = startDate.toISOString();
effectiveEndDate = endDate.toISOString();
} else if (options.timeframe && options.timeframeUnit) {
const now = new Date();
const startDate = new Date();
switch (options.timeframeUnit) {
case 'hours':
startDate.setHours(now.getHours() - options.timeframe);
break;
case 'days':
startDate.setDate(now.getDate() - options.timeframe);
break;
case 'weeks':
startDate.setDate(now.getDate() - (options.timeframe * 7));
break;
case 'months':
startDate.setMonth(now.getMonth() - options.timeframe);
break;
}
effectiveStartDate = startDate.toISOString();
effectiveEndDate = now.toISOString();
}
// CURSOR-BASED PAGINATION
const limit = Math.min(options.limit || 100, 100);
// Decode cursor if provided
let cursorData: any = null;
if (options.cursor) {
try {
cursorData = JSON.parse(Buffer.from(options.cursor, 'base64').toString('utf-8'));
logger.debug(`[DataCatalogService] Using cursor:`, cursorData);
} catch (e) {
console.error('[DataCatalogService] Invalid cursor format:', e);
throw new Error('Invalid cursor format');
}
}
// Single space with cursor - direct API call
if (options.spaceId && cursorData) {
const params = new URLSearchParams();
params.set('spaceId', options.spaceId);
params.set('limit', String(limit));
params.set('includeResourceAttributes', 'true');
// Cursor-based filtering - API'nin desteklediği parametreler
if (cursorData.lastItemId) {
// API afterId parametresini destekliyorsa kullan, yoksa offset kullan
params.set('offset', String(cursorData.offset || 0));
}
const url = `/items?${params.toString()}`;
const response = await this.apiClient.makeRequest(url, 'GET');
const items = response.data || response || [];
items.forEach((item: any) => {
item.resourceSubType = item.resourceAttributes?.resourceSubType || item.resourceSubType;
});
const space = await this.getSpaceDetails(options.spaceId, false, false);
if (!space) {
return this.createEmptyResult(options);
}
const enhancedItems = items.map((item: any) => ({
...item,
_spaceId: space.id,
_spaceName: space.name,
_spaceType: space.type,
_spaceOwner: space.spaceInfo.ownerName,
_spaceOwnerId: space.spaceInfo.ownerId
}));
// Generate next cursor
let nextCursor: string | undefined;
const hasMore = items.length === limit;
if (hasMore && items.length > 0) {
const lastItem = items[items.length - 1];
const newOffset = (cursorData.offset || 0) + items.length;
nextCursor = Buffer.from(JSON.stringify({
lastItemId: lastItem.id,
offset: newOffset,
sortBy: options.sortBy || 'modified',
sortOrder: options.sortOrder || 'desc'
})).toString('base64');
}
const result = this.buildFlatResult(enhancedItems, options);
result.metadata = {
totalItems: -1,
returnedItems: items.length,
cursor: nextCursor,
hasMore: hasMore,
searchTime: Date.now() - startTime,
spacesSearched: 1,
filters: options,
offset: 0,
limit: limit
};
return result;
}
// Backward compatibility - offset kullanılıyorsa
if (options.offset && options.offset > 0 && !options.cursor && options.spaceId) {
logger.debug(`[DataCatalogService] DEPRECATED: Using offset pagination, please use cursor`);
const url = `/items?spaceId=${options.spaceId}&limit=${limit}&offset=${options.offset}&includeResourceAttributes=true`;
const response = await this.apiClient.makeRequest(url, 'GET');
const items = response.data || response || [];
items.forEach((item: any) => {
item.resourceSubType = item.resourceAttributes?.resourceSubType || item.resourceSubType;
});
const space = await this.getSpaceDetails(options.spaceId, false, false);
if (!space) {
return this.createEmptyResult(options);
}
const enhancedItems = items.map((item: any) => ({
...item,
_spaceId: space.id,
_spaceName: space.name,
_spaceType: space.type,
_spaceOwner: space.spaceInfo.ownerName,
_spaceOwnerId: space.spaceInfo.ownerId
}));
const result = this.buildFlatResult(enhancedItems, options);
const hasMore = items.length === limit;
// Generate cursor for next page
let nextCursor: string | undefined;
if (hasMore && items.length > 0) {
const newOffset = options.offset + items.length;
nextCursor = Buffer.from(JSON.stringify({
offset: newOffset,
sortBy: options.sortBy || 'modified',
sortOrder: options.sortOrder || 'desc'
})).toString('base64');
}
result.metadata = {
totalItems: -1,
returnedItems: items.length,
cursor: nextCursor,
offset: options.offset,
limit: limit,
hasMore: hasMore,
searchTime: Date.now() - startTime,
spacesSearched: 1,
filters: options
};
return result;
}
// First page - no cursor or offset
let targetSpaces: QlikSpace[] = [];
if (options.spaceId) {
const space = await this.getSpaceDetails(options.spaceId, false, false);
if (space) {
targetSpaces = [space];
} else {
return this.createEmptyResult(options);
}
} else if (options.spaceIds && options.spaceIds.length > 0) {
const spacePromises = options.spaceIds.map(id =>
this.getSpaceDetails(id, false, false)
);
const spaces = await Promise.all(spacePromises);
targetSpaces = spaces.filter(s => s !== null) as QlikSpace[];
} else if (options.allSpaces) {
const catalogOptions: SpaceCatalogSearchOptions = {
includeMembers: false,
includeItems: false,
limit: 1000
};
if (options.spaceType && options.spaceType !== 'all') {
catalogOptions.spaceType = options.spaceType;
}
if (options.hasDataAssets !== undefined) {
catalogOptions.hasDataAssets = options.hasDataAssets;
}
const catalog = await this.getSpacesCatalog(catalogOptions);
targetSpaces = catalog.spaces;
} else {
return this.createEmptyResult(options);
}
logger.debug(`[DataCatalogService] Searching ${targetSpaces.length} spaces`);
// Collect items from spaces
const allItems: EnhancedSpaceItem[] = [];
for (const space of targetSpaces) {
const { items } = await this.fetchAllSpaceItems(space.id);
const enhancedItems = items.map((item: any) => ({
...item,
_spaceId: space.id,
_spaceName: space.name,
_spaceType: space.type,
_spaceOwner: space.spaceInfo.ownerName,
_spaceOwnerId: space.spaceInfo.ownerId
}));
allItems.push(...enhancedItems);
}
logger.debug(`[DataCatalogService] Found ${allItems.length} total items`);
// Resolve owner if needed
let resolvedOwnerId = options.ownerId;
if (!resolvedOwnerId && (options.ownerName || options.ownerEmail)) {
try {
const users = await this.apiClient.searchUsers(
options.ownerName || options.ownerEmail || ''
);
if (users && users.length > 0) {
const targetUser = users.find((u: any) =>
(options.ownerName && u.name?.toLowerCase() === options.ownerName.toLowerCase()) ||
(options.ownerEmail && u.email?.toLowerCase() === options.ownerEmail.toLowerCase())
) || users[0];
resolvedOwnerId = targetUser.id;
logger.debug(`[DataCatalogService] Resolved owner: ${targetUser.name} (${resolvedOwnerId})`);
}
} catch (error) {
console.error('[DataCatalogService] Failed to resolve owner:', error);
}
}
// Apply filters
let filteredItems = [...allItems];
// Date filtering
if (effectiveStartDate || effectiveEndDate) {
const dateField = options.dateField || 'modified';
filteredItems = filteredItems.filter(item => {
let itemDate: Date;
if (dateField === 'created') {
const dateStr = item.createdDate;
itemDate = dateStr ? new Date(dateStr) : new Date(0);
} else {
const dateStr = item.modifiedDate;
itemDate = dateStr ? new Date(dateStr) : new Date(0);
}
if (itemDate.getTime() === 0) {
return false;
}
if (effectiveStartDate && itemDate < new Date(effectiveStartDate)) {
return false;
}
if (effectiveEndDate && itemDate > new Date(effectiveEndDate)) {
return false;
}
return true;
});
}
// Owner filtering
if (resolvedOwnerId) {
filteredItems = filteredItems.filter(item => {
const itemOwnerId = (typeof item.owner === 'object' ? item.owner?.id : item.owner);
return itemOwnerId === resolvedOwnerId;
});
}
// Type filtering
if (options.itemTypes && options.itemTypes.length > 0) {
const typesLower = options.itemTypes.map(t => t.toLowerCase());
filteredItems = filteredItems.filter(item => {
const itemType = this.normalizeItemType(
item.resourceType || item.type,
item.name,
item.resourceSubType
);
return typesLower.includes(itemType.toLowerCase());
});
}
// Query filtering
if (options.query) {
filteredItems = this.searchItemsByQuery(filteredItems, options.query);
}
// Sort
if (options.sortBy) {
filteredItems = this.sortItems(filteredItems, options.sortBy, options.sortOrder);
} else if (effectiveStartDate || effectiveEndDate) {
filteredItems = this.sortItems(filteredItems, 'modified', 'desc');
}
// Pagination
const totalBeforePagination = filteredItems.length;
const offset = options.offset || 0;
const paginatedItems = options.skipPagination
? filteredItems
: filteredItems.slice(offset, offset + limit);
const hasMore = totalBeforePagination > offset + limit;
// Generate cursor for next page
let nextCursor: string | undefined;
if (hasMore && paginatedItems.length > 0) {
const newOffset = offset + paginatedItems.length;
nextCursor = Buffer.from(JSON.stringify({
offset: newOffset,
sortBy: options.sortBy || 'modified',
sortOrder: options.sortOrder || 'desc'
})).toString('base64');
}
logger.debug(`[DataCatalogService] Returning ${paginatedItems.length} of ${totalBeforePagination} total items`);
// Enhance with owner info if requested
if (options.includeOwnerInfo && paginatedItems.length > 0) {
await this.enhanceItemsWithOwnerInfo(paginatedItems);
}
// Build result based on grouping options
let result: SpaceItemsResult;
if (options.groupBySpace) {
result = this.buildSpaceGroupedResult(paginatedItems, targetSpaces, options);
} else if (options.groupByType) {
result = this.buildTypeGroupedResult(paginatedItems, options);
} else {
result = this.buildFlatResult(paginatedItems, options);
}
// Add metadata
result.metadata = {
totalItems: totalBeforePagination,
returnedItems: paginatedItems.length,
cursor: nextCursor,
offset: offset,
limit: limit,
hasMore: hasMore,
searchTime: Date.now() - startTime,
spacesSearched: targetSpaces.length,
filters: options
};
return result;
} catch (error) {
console.error('[DataCatalogService] Failed to get space items:', error);
return {
success: false,
error: error instanceof Error ? error.message : 'Unknown error'
};
}
}
/**
* Search items by query using token matching
*/
private searchItemsByQuery(items: any[], query: string): any[] {
const queryLower = query.toLowerCase();
const queryTokens = queryLower.split(/[\s,._\-\/]+/).filter(t => t.length > 0);
return items.filter(item => {
const searchableText = [
item.name,
item.description,
item.type,
item._spaceName,
item.tags?.join(' ')
].filter(Boolean).join(' ').toLowerCase();
// All tokens must match
return queryTokens.every(token => searchableText.includes(token));
});
}
/**
* Sort items by specified field - FIX: Add resourceSubType
*/
private sortItems(items: any[], sortBy: string, sortOrder?: string): any[] {
const order = sortOrder === 'desc' ? -1 : 1;
return [...items].sort((a, b) => {
switch (sortBy) {
case 'name':
return order * (a.name || '').localeCompare(b.name || '');
case 'created':
const aCreated = a.createdAt || a.created || a.createdDate || 0;
const bCreated = b.createdAt || b.created || b.createdDate || 0;
return order * (new Date(aCreated).getTime() - new Date(bCreated).getTime());
case 'modified':
const aModified = a.updatedAt || a.modified || a.modifiedDate || 0;
const bModified = b.updatedAt || b.modified || b.modifiedDate || 0;
return order * (new Date(aModified).getTime() - new Date(bModified).getTime());
case 'size':
return order * ((a.size || 0) - (b.size || 0));
case 'type':
const aType = this.normalizeItemType(
a.resourceType || a.type,
a.name,
(a as any).resourceSubType // Type assertion
);
const bType = this.normalizeItemType(
b.resourceType || b.type,
b.name,
(b as any).resourceSubType // Type assertion
);
return order * aType.localeCompare(bType);
default:
return 0;
}
});
}
/**
* Build flat result structure
*/
private buildFlatResult(items: any[], options: SpaceItemsOptions): SpaceItemsResult {
return {
success: true,
items: items.map(item => ({
id: item.id || item.itemId,
name: item.name,
normalizeItemType: this.normalizeItemType(
item.resourceType || item.type,
item.name,
(item as any).resourceSubType // Type assertion
),
resourceType: item.resourceType || item.type,
owner: item.ownerName || item.owner?.name || 'Unknown',
ownerId: item.ownerId || item.owner?.id,
created: item.createdAt || item.created || item.createdDate,
modified: item.updatedAt || item.modified || item.modifiedDate,
size: item.size,
sizeFormatted: this.formatFileSize(item.size || 0),
description: item.description,
tags: item.tags || [],
...(options.includeSpaceInfo !== false ?
{
space: {
id: item._spaceId,
name: item._spaceName,
type: item._spaceType,
owner: item._spaceOwner
}
} : {})
}))
};
}
/**
* Build space-grouped result structure
*/
private buildSpaceGroupedResult(
items: any[],
spaces: QlikSpace[],
options: SpaceItemsOptions
): SpaceItemsResult {
const itemsBySpace = new Map<string, any[]>();
// Group items by space
items.forEach(item => {
const spaceId = item._spaceId;
if (!itemsBySpace.has(spaceId)) {
itemsBySpace.set(spaceId, []);
}
itemsBySpace.get(spaceId)!.push(item);
});
return {
success: true,
groupedBySpace: spaces
.filter(space => itemsBySpace.has(space.id))
.map(space => ({
space: {
id: space.id,
name: space.name,
type: space.type,
owner: space.spaceInfo.ownerName
},
itemCount: itemsBySpace.get(space.id)?.length || 0,
items: this.buildFlatResult(itemsBySpace.get(space.id) || [], options).items!
}))
};
}
/**
* Build type-grouped result structure
*/
private buildTypeGroupedResult(items: any[], options: SpaceItemsOptions): SpaceItemsResult {
const itemsByType = new Map<string, any[]>();
// Group items by type
items.forEach(item => {
// CHANGE: Pass resourceSubType as the third parameter
const type = this.normalizeItemType(
item.resourceType || item.type,
item.name,
item.resourceSubType // ADD THIS
);
if (!itemsByType.has(type)) {
itemsByType.set(type, []);
}
itemsByType.get(type)!.push(item);
});
// Sort types by count (descending)
const sortedTypes = Array.from(itemsByType.entries())
.sort((a, b) => b[1].length - a[1].length);
return {
success: true,
groupedByType: sortedTypes.map(([type, typeItems]) => ({
type: type,
count: typeItems.length,
items: this.buildFlatResult(typeItems, options).items!
}))
};
}
/**fetchAllSpaceItems
* Fetch all items from a space with pagination
*/
private async fetchAllSpaceItems(spaceId: string, maxLimit: number = 10000): Promise<{ items: any[], dataAssets: any[] }> {
const allItems: any[] = [];
const seenIds = new Set<string>();
let hasMore = true;
let iteration = 0;
const maxIterations = 100;
const batchSize = 100; // Size per API call
console.error(`[DataCatalogService] Starting to fetch all items for space ${spaceId}`);
console.error('[DataCatalogService] Using cursor-based pagination initially');
// Start with cursor-based pagination
let nextUrl: string | null = null;
while (hasMore && iteration < maxIterations) {
iteration++;
let url: string;
if (nextUrl) {
// Use the next URL from the API response
url = nextUrl.startsWith('http') ? nextUrl : nextUrl;
} else if (iteration === 1) {
// Initial request
url = `/items?spaceId=${spaceId}&limit=${batchSize}&includeResourceAttributes=true`;
console.error(`[DataCatalogService] Initial fetch for space ${spaceId}`);
} else {
// This shouldn't happen with proper cursor pagination
console.error('[DataCatalogService] No next URL available, stopping');
break;
}
try {
const response = await this.apiClient.makeRequest(
url.startsWith('http') ? url : url,
'GET'
);
// Debug response structure
if (iteration === 1) {
console.error(`[DataCatalogService] Response structure:`, {
hasData: !!response.data,
hasItems: !!(response.data?.items || response.items),
isArray: Array.isArray(response.data || response),
hasLinks: !!(response.links || response.data?.links),
hasPagination: !!(response.pagination || response.data?.pagination),
itemCount: Array.isArray(response.data || response)
? (response.data || response).length
: (response.data?.items || response.items || response.data || response)?.length
});
// Check if API uses offset-based pagination
if (response.links?.next?.href?.includes('next=')) {
console.error('[DataCatalogService] API uses offset-based pagination, switching...');
}
}
// Extract items from various possible response structures
let items: any[] = [];
if (Array.isArray(response)) {
items = response;
} else if (Array.isArray(response.data)) {
items = response.data;
} else if (response.data?.items) {
items = response.data.items;
} else if (response.items) {
items = response.items;
} else if (response.data && typeof response.data === 'object') {
items = Object.values(response.data);
}
// Process items
let newItemsCount = 0;
items.forEach((item: any) => {
// Store resourceSubType
item.resourceSubType = item.resourceAttributes?.resourceSubType || item.resourceSubType;
if (item.id && !seenIds.has(item.id)) {
seenIds.add(item.id);
allItems.push(item);
newItemsCount++;
}
});
console.error(`[DataCatalogService] Iteration ${iteration}: returned ${items.length} items, ${newItemsCount} new, total: ${allItems.length}`);
// Check for next page
nextUrl = null;
// Try to find next URL in various formats
if (response.links?.next?.href) {
nextUrl = response.links.next.href;
console.error('[DataCatalogService] Found next URL in links.next.href');
} else if (response.data?.links?.next?.href) {
nextUrl = response.data.links.next.href;
console.error('[DataCatalogService] Found next URL in data.links.next.href');
} else if (response.links?.next) {
nextUrl = response.links.next;
console.error('[DataCatalogService] Found next URL in links.next');
} else if (response.next) {
nextUrl = response.next;
console.error('[DataCatalogService] Found next URL in next');
}
// If we have a next URL, continue
if (nextUrl) {
// Extract cursor from the URL if it's a full URL
if (nextUrl.includes('next=')) {
const urlObj = new URL(nextUrl, 'https://atlas.eu.qlikcloud.com');
const nextCursor = urlObj.searchParams.get('next');
if (nextCursor) {
nextUrl = `/items?spaceId=${spaceId}&limit=${batchSize}&includeResourceAttributes=true&next=${nextCursor}`;
}
} else if (!nextUrl.startsWith('/')) {
nextUrl = `/items?${nextUrl}`;
}
console.error('[DataCatalogService] Using next URL from API:', nextUrl);
hasMore = true;
} else {
// No next URL means we're done
console.error('[DataCatalogService] No next URL, done');
hasMore = false;
}
// Additional safety checks
if (items.length === 0) {
console.error('[DataCatalogService] No items in response, stopping');
hasMore = false;
} else if (newItemsCount === 0 && iteration > 1) {
console.error('[DataCatalogService] No new items found, likely at end');
hasMore = false;
}
} catch (error: any) {
console.error(`[DataCatalogService] Error fetching items:`, error.message);
// Try to continue if we already have items
if (allItems.length > 0) {
console.error(`[DataCatalogService] Continuing despite error, already have ${allItems.length} items`);
hasMore = false;
} else {
throw error;
}
}
}
console.error(`[DataCatalogService] Completed fetching items for space ${spaceId}:`);
console.error(`[DataCatalogService] - Total unique items: ${allItems.length}`);
console.error(`[DataCatalogService] - Total iterations: ${iteration}`);
console.error(`[DataCatalogService] - Pagination method: ${iteration > 1 ? 'offset' : 'single page'}`);
// Separate data assets
const dataAssets = allItems.filter(item => {
const type = this.normalizeItemType(
item.resourceType || item.type,
item.name,
item.resourceSubType
);
return type === 'dataset';
});
console.error(`[DataCatalogService] Found ${dataAssets.length} data assets out of ${allItems.length} total items`);
return { items: allItems, dataAssets };
}
/**
* Create empty result when no items found
*/
private createEmptyResult(options: SpaceItemsOptions): SpaceItemsResult {
const result: SpaceItemsResult = {
success: true,
metadata: {
totalItems: 0,
returnedItems: 0,
offset: options.offset || 0,
limit: options.limit || 100,
hasMore: false,
searchTime: 0,
spacesSearched: 0,
filters: {
query: options.query,
ownerId: options.ownerId,
itemTypes: options.itemTypes,
spaceType: options.spaceType
}
}
};
if (options.groupBySpace) {
result.groupedBySpace = [];
} else if (options.groupByType) {
result.groupedByType = [];
} else {
result.items = [];
}
return result;
}
/**
* Enhance items with owner information
*/
private async enhanceItemsWithOwnerInfo(items: any[]): Promise<void> {
const ownerIds = [...new Set(items
.map(item => item.ownerId || item.owner?.id)
.filter(Boolean))] as string[];
if (ownerIds.length === 0) return;
const resolvedUsers = await this.apiClient.resolveOwnersToUsers(ownerIds);
items.forEach(item => {
const ownerId = item.ownerId || item.owner?.id;
if (ownerId && resolvedUsers.has(ownerId)) {
const user = resolvedUsers.get(ownerId)!;
item.ownerName = user.displayName;
item.ownerEmail = user.email;
}
});
}
private getItemDate(item: any, dateField: 'created' | 'modified'): Date {
if (dateField === 'created') {
// For SpaceItem interface, the field is createdDate
const dateValue = item.createdDate || null;
return dateValue ? new Date(dateValue) : new Date(0);
} else {
// For SpaceItem interface, the field is modifiedDate
const dateValue = item.modifiedDate || null;
return dateValue ? new Date(dateValue) : new Date(0);
}
}
/**
* Format file size helper
*/
private formatFileSize(bytes: number): string {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
}
