import { JsmClient } from '../api/jsmClient.js';
import { AqlSearchRequest, AqlSearchResponse, JsmObject } from '../types/index.js';
import { logDebug } from './index.js';
export interface PaginationOptions {
/** Maximum number of pages to fetch (safety limit) */
maxPages?: number;
/** Page size (objects per request) */
pageSize?: number;
}
export interface PaginatedResult {
/** All objects from all pages */
values: JsmObject[];
/** Total pages fetched */
pagesFetched: number;
/** Total objects retrieved */
totalRetrieved: number;
/** Whether pagination was stopped due to maxPages limit */
limitReached: boolean;
/** API reported total (may be unreliable) */
apiReportedTotal?: number;
}
/**
* Robust pagination helper that uses the reliable pattern:
* - Request N objects
* - If exactly N objects returned, assume more pages exist
* - Continue until returned count < requested count
*
* This avoids relying on potentially unreliable API metadata (total, isLast)
*/
export async function paginateAqlSearch(
client: JsmClient,
baseRequest: Omit<AqlSearchRequest, 'startAt' | 'maxResults'>,
options: PaginationOptions = {}
): Promise<PaginatedResult> {
const {
maxPages = 10, // Default safety limit
pageSize = 1000
} = options;
const allValues: JsmObject[] = [];
let currentPage = 0;
let startAt = 0;
let limitReached = false;
let apiReportedTotal: number | undefined;
logDebug('Starting robust pagination', {
query: baseRequest.qlQuery,
maxPages,
pageSize
});
while (currentPage < maxPages) {
currentPage++;
logDebug(`Fetching page ${currentPage}`, { startAt, pageSize });
const request: AqlSearchRequest = {
...baseRequest,
startAt,
maxResults: pageSize
};
const response: AqlSearchResponse = await client.searchAssetsAql(request);
// Store API reported total from first response (for reference only)
if (apiReportedTotal === undefined) {
apiReportedTotal = response.total;
}
const returnedCount = response.values?.length || 0;
logDebug(`Page ${currentPage} results`, {
returned: returnedCount,
requested: pageSize,
apiTotal: response.total,
apiIsLast: response.isLast
});
// Add results to collection
if (response.values && response.values.length > 0) {
allValues.push(...response.values);
}
// ROBUST PAGINATION LOGIC:
// If we got fewer objects than requested, we've reached the end
if (returnedCount < pageSize) {
logDebug('Pagination complete - returned count < requested count', {
returnedCount,
pageSize,
totalPages: currentPage,
totalObjects: allValues.length
});
break;
}
// If we got exactly the requested count, there might be more pages
startAt += pageSize;
}
// Check if we stopped due to maxPages limit
if (currentPage >= maxPages) {
const lastPageRequest: AqlSearchRequest = {
...baseRequest,
startAt,
maxResults: pageSize
};
try {
const testResponse = await client.searchAssetsAql(lastPageRequest);
if (testResponse.values && testResponse.values.length > 0) {
limitReached = true;
logDebug('Pagination stopped due to maxPages limit', {
maxPages,
moreDataAvailable: true
});
}
} catch (error) {
logDebug('Error testing for more pages - assuming end reached', error);
}
}
const result: PaginatedResult = {
values: allValues,
pagesFetched: currentPage,
totalRetrieved: allValues.length,
limitReached,
apiReportedTotal
};
logDebug('Pagination completed', {
pagesFetched: result.pagesFetched,
totalRetrieved: result.totalRetrieved,
limitReached: result.limitReached,
apiReportedTotal: result.apiReportedTotal
});
return result;
}
/**
* Format pagination information for user display
*/
export function formatPaginationInfo(result: PaginatedResult): string {
let info = `Retrieved ${result.totalRetrieved} objects`;
if (result.pagesFetched > 1) {
info += ` across ${result.pagesFetched} pages`;
}
if (result.limitReached) {
info += ` (pagination limit reached - more data may be available)`;
}
// Show API reported total for comparison (if different and reliable)
if (result.apiReportedTotal &&
result.apiReportedTotal !== result.totalRetrieved &&
!result.limitReached) {
info += ` (API reported total: ${result.apiReportedTotal})`;
}
return info;
}
