import * as jsforce from 'jsforce';
import { SalesforceConnection } from './connection';
import logger from '../utils/logger';
import { AppError } from '../utils/errorHandler';
import { RateLimiter, RateLimitConfig } from '../utils/rateLimiter';
import { IncrementalTracker } from '../utils/incrementalTracker';
export interface MetadataType {
type: string;
folder?: string;
}
export interface MetadataInfo {
id: string;
fullName: string;
type: string;
createdDate: string;
lastModifiedDate: string;
fileName?: string;
}
export interface RetrieveResult {
success: boolean;
zipFile?: string;
fileProperties?: any[];
messages?: any[];
}
export class SalesforceMetadataClient {
private connection: SalesforceConnection;
private conn: jsforce.Connection | null = null;
private rateLimiter: RateLimiter;
private incrementalTracker: IncrementalTracker | null = null;
constructor(connection: SalesforceConnection, rateLimitConfig?: RateLimitConfig) {
this.connection = connection;
// Default rate limiting configuration for Salesforce
const defaultConfig: RateLimitConfig = {
maxRequestsPerSecond: 5,
maxRequestsPerHour: 5000,
retryAttempts: 3,
baseDelay: 1000
};
this.rateLimiter = new RateLimiter(rateLimitConfig || defaultConfig);
}
async authenticate(): Promise<void> {
try {
// Use OAuth authentication by default
const username = process.env.SF_USERNAME;
const password = process.env.SF_PASSWORD;
const securityToken = process.env.SF_SECURITY_TOKEN;
if (!username || !password || !securityToken) {
throw new AppError('Missing Salesforce credentials', 400);
}
this.conn = await this.connection.connectWithOAuth(username, password, securityToken);
logger.info('Successfully authenticated with Salesforce');
} catch (error) {
logger.error('Failed to authenticate with Salesforce', { error });
throw new AppError('Salesforce authentication failed', 401);
}
}
async listMetadata(types: MetadataType[]): Promise<MetadataInfo[]> {
if (!this.conn) {
throw new AppError('Not authenticated', 401);
}
try {
logger.debug('Listing metadata types', { types });
// Implement rate limiting by batching requests
const batchSize = 3; // Salesforce allows up to 3 types per call
const results: MetadataInfo[] = [];
for (let i = 0; i < types.length; i += batchSize) {
const batch = types.slice(i, i + batchSize);
const batchResults = await this.rateLimiter.executeWithRateLimit(async () => {
return await this.conn!.metadata.list(batch);
});
// Handle single result or array
const resultArray = Array.isArray(batchResults) ? batchResults : [batchResults];
results.push(...resultArray);
}
logger.info(`Retrieved ${results.length} metadata items`);
return results;
} catch (error) {
logger.error('Failed to list metadata', { error, types });
throw new AppError('Failed to retrieve metadata list', 500);
}
}
async readMetadata(type: string, fullNames: string[]): Promise<any[]> {
if (!this.conn) {
throw new AppError('Not authenticated', 401);
}
try {
logger.debug('Reading metadata', { type, count: fullNames.length });
// Batch requests to stay within limits
const batchSize = 10;
const results: any[] = [];
for (let i = 0; i < fullNames.length; i += batchSize) {
const batch = fullNames.slice(i, i + batchSize);
const batchResults = await this.rateLimiter.executeWithRateLimit(async () => {
return await this.conn!.metadata.read({ type } as any, batch);
});
// Handle single result or array
const resultArray = Array.isArray(batchResults) ? batchResults : [batchResults];
results.push(...resultArray);
}
logger.info(`Read ${results.length} ${type} metadata items`);
return results;
} catch (error) {
logger.error('Failed to read metadata', { error, type, fullNames });
throw new AppError(`Failed to read ${type} metadata`, 500);
}
}
async retrievePackage(packageXml: string): Promise<RetrieveResult> {
if (!this.conn) {
throw new AppError('Not authenticated', 401);
}
try {
logger.debug('Starting package retrieval');
const retrieveRequest = await this.rateLimiter.executeWithRateLimit(async () => {
return await this.conn!.metadata.retrieve({
unpackaged: JSON.parse(packageXml)
} as any);
});
if (!retrieveRequest.id) {
throw new AppError('Failed to initiate retrieve request', 500);
}
const result = await this.pollRetrieveStatus(retrieveRequest.id);
logger.info('Package retrieval completed', { success: result.success });
return result;
} catch (error) {
logger.error('Failed to retrieve package', { error });
throw new AppError('Package retrieval failed', 500);
}
}
private async pollRetrieveStatus(retrieveId: string): Promise<RetrieveResult> {
const maxAttempts = 30;
let attempts = 0;
let delay = 1000; // Start with 1 second
while (attempts < maxAttempts) {
try {
const result = await this.conn!.metadata.checkRetrieveStatus(retrieveId);
if (result.done) {
return result;
}
attempts++;
logger.debug(`Retrieve status check ${attempts}/${maxAttempts}`, {
retrieveId,
done: result.done
});
// Exponential backoff with jitter
await this.delay(delay + Math.random() * 1000);
delay = Math.min(delay * 1.5, 10000); // Cap at 10 seconds
} catch (error) {
logger.error('Error checking retrieve status', { error, retrieveId, attempts });
throw new AppError('Failed to check retrieve status', 500);
}
}
throw new AppError('Retrieve operation timed out', 408);
}
private delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
getConnection(): jsforce.Connection {
if (!this.conn) {
throw new AppError('Not authenticated', 401);
}
return this.conn;
}
async initializeIncrementalTracking(orgId: string): Promise<void> {
this.incrementalTracker = new IncrementalTracker(orgId);
await this.incrementalTracker.loadState();
logger.info('Initialized incremental tracking', { orgId });
}
async listMetadataIncremental(types: MetadataType[]): Promise<{
all: MetadataInfo[];
changes: {
added: MetadataInfo[];
modified: MetadataInfo[];
deleted: any[];
unchanged: MetadataInfo[];
}
}> {
if (!this.incrementalTracker) {
throw new AppError('Incremental tracking not initialized', 500);
}
// Get all current metadata
const allMetadata = await this.listMetadata(types);
// Detect changes
const changes = this.incrementalTracker.detectChanges(allMetadata);
// Update snapshots with current state
this.incrementalTracker.updateSnapshots(allMetadata);
await this.incrementalTracker.saveState();
return {
all: allMetadata,
changes
};
}
async getChangedMetadataSince(sinceDate: Date): Promise<MetadataInfo[]> {
if (!this.incrementalTracker) {
throw new AppError('Incremental tracking not initialized', 500);
}
const changedSnapshots = this.incrementalTracker.getChangedSince(sinceDate);
// Convert snapshots back to MetadataInfo format
return changedSnapshots.map(snapshot => ({
id: '',
fullName: snapshot.fullName,
type: snapshot.type,
createdDate: '',
lastModifiedDate: snapshot.lastModifiedDate
}));
}
async performIncrementalSync(types: MetadataType[]): Promise<{
totalItems: number;
newItems: number;
modifiedItems: number;
deletedItems: number;
}> {
if (!this.incrementalTracker) {
throw new AppError('Incremental tracking not initialized', 500);
}
logger.info('Starting incremental metadata sync');
const result = await this.listMetadataIncremental(types);
const stats = {
totalItems: result.all.length,
newItems: result.changes.added.length,
modifiedItems: result.changes.modified.length,
deletedItems: result.changes.deleted.length
};
logger.info('Incremental sync completed', stats);
return stats;
}
getRateLimitStats() {
return this.rateLimiter.getStats();
}
}
