/**
* Batch Processor
*
* Handles batch file operations for processing 100+ files efficiently.
* Implements intelligent batching with error handling and progress tracking.
*/
import { logger } from '../utils/logger.js';
import { retryHandler } from '../utils/retry-handler.js';
import { gitOperations } from './git-operations.js';
/**
* Batch Processor Class
*/
export class BatchProcessor {
/**
* Process files in batches
* @param {Object} params - Batch processing parameters
* @param {Object} env - Environment variables
* @param {string} correlationId - Request correlation ID
* @returns {Promise<Object>} Processing result
*/
async process(params, env, correlationId) {
const { owner, repo, branch, batches, message } = params;
let currentSha = await this.getCurrentBranchSha(env.GITHUB_TOKEN, owner, repo, branch, correlationId);
const initialSha = currentSha;
const results = {
totalBatches: batches.length,
successfulBatches: 0,
failedBatches: 0,
totalFiles: batches.reduce((sum, batch) => sum + batch.length, 0),
processedFiles: 0,
commits: [],
errors: []
};
logger.info('Starting batch processing', {
correlationId,
totalBatches: batches.length,
totalFiles: results.totalFiles,
initialSha
});
for (let i = 0; i < batches.length; i++) {
const batch = batches[i];
const batchNumber = i + 1;
logger.info('Processing batch', {
correlationId,
batchNumber,
filesInBatch: batch.length,
currentSha
});
try {
const result = await this.processBatch(
env.GITHUB_TOKEN,
owner,
repo,
branch,
batch,
message,
currentSha,
correlationId,
batchNumber
);
results.successfulBatches++;
results.processedFiles += batch.length;
results.commits.push(result);
currentSha = result.sha;
logger.info('Batch processed successfully', {
correlationId,
batchNumber,
commitSha: result.sha,
filesProcessed: batch.length
});
} catch (error) {
results.failedBatches++;
logger.error('Batch processing failed', {
correlationId,
batchNumber,
error: error.message
});
results.errors.push({
batchNumber,
error: error.message,
files: batch.length
});
// Stop processing on error to maintain consistency
// Rollback to initial state
try {
await this.rollbackToSha(
env.GITHUB_TOKEN,
owner,
repo,
branch,
initialSha,
correlationId
);
logger.info('Rolled back to initial state', {
correlationId,
batchNumber,
rollbackSha: initialSha
});
} catch (rollbackError) {
logger.error('Rollback failed', {
correlationId,
error: rollbackError.message
});
}
throw new Error(`Batch ${batchNumber} failed: ${error.message}. Rolled back to initial state.`);
}
}
logger.info('Batch processing completed', {
correlationId,
totalBatches: results.totalBatches,
successfulBatches: results.successfulBatches,
failedBatches: results.failedBatches,
totalFiles: results.totalFiles,
finalSha: currentSha
});
return {
...results,
finalSha: currentSha,
initialSha
};
}
/**
* Process a single batch
* @param {string} githubToken - GitHub access token
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {string} branch - Branch name
* @param {Array} batch - Files in batch
* @param {string} message - Commit message
* @param {string} currentSha - Current commit SHA
* @param {string} correlationId - Request correlation ID
* @param {number} batchNumber - Batch number
* @returns {Promise<Object>} Batch result
*/
async processBatch(githubToken, owner, repo, branch, batch, message, currentSha, correlationId, batchNumber) {
// Create tree with file changes
const treeResponse = await retryHandler.execute(
() => fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees`, {
method: 'POST',
headers: {
'Authorization': `token ${githubToken}`,
'User-Agent': 'github-mcp-control-plane',
'Accept': 'application/vnd.github.v3+json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
base_tree: currentSha,
tree: batch.map(file => ({
path: file.path,
mode: '100644',
type: 'blob',
content: file.content
}))
})
}),
{ correlationId, operation: 'create_tree_batch' }
);
if (!treeResponse.ok) {
const error = await treeResponse.json();
throw new Error(`Failed to create tree for batch ${batchNumber}: ${treeResponse.status} ${error.message}`);
}
const treeData = await treeResponse.json();
// Create commit with batch message
const batchMessage = batch.length > 1
? `${message} (batch ${batchNumber}/${batchNumber})`
: message;
const commitResponse = await retryHandler.execute(
() => fetch(`https://api.github.com/repos/${owner}/${repo}/git/commits`, {
method: 'POST',
headers: {
'Authorization': `token ${githubToken}`,
'User-Agent': 'github-mcp-control-plane',
'Accept': 'application/vnd.github.v3+json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
message: batchMessage,
tree: treeData.sha,
parents: [currentSha]
})
}),
{ correlationId, operation: 'create_commit_batch' }
);
if (!commitResponse.ok) {
const error = await commitResponse.json();
throw new Error(`Failed to create commit for batch ${batchNumber}: ${commitResponse.status} ${error.message}`);
}
const commitData = await commitResponse.json();
// Update branch reference
const updateResponse = await retryHandler.execute(
() => fetch(`https://api.github.com/repos/${owner}/${repo}/git/refs/heads/${branch}`, {
method: 'PATCH',
headers: {
'Authorization': `token ${githubToken}`,
'User-Agent': 'github-mcp-control-plane',
'Accept': 'application/vnd.github.v3+json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
sha: commitData.sha,
force: false
})
}),
{ correlationId, operation: 'update_branch_batch' }
);
if (!updateResponse.ok) {
const error = await updateResponse.json();
throw new Error(`Failed to update branch for batch ${batchNumber}: ${updateResponse.status} ${error.message}`);
}
return {
sha: commitData.sha,
message: commitData.message,
tree: commitData.tree.sha,
parent: currentSha,
batchNumber,
filesProcessed: batch.length,
url: commitData.html_url
};
}
/**
* Get current branch SHA
* @param {string} githubToken - GitHub access token
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {string} branch - Branch name
* @param {string} correlationId - Request correlation ID
* @returns {Promise<string>} Current commit SHA
*/
async getCurrentBranchSha(githubToken, owner, repo, branch, correlationId) {
const refData = await gitOperations.getRef(githubToken, owner, repo, `heads/${branch}`, correlationId);
return refData.object.sha;
}
/**
* Rollback branch to specific SHA
* @param {string} githubToken - GitHub access token
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {string} branch - Branch name
* @param {string} sha - SHA to rollback to
* @param {string} correlationId - Request correlation ID
* @returns {Promise<void>}
*/
async rollbackToSha(githubToken, owner, repo, branch, sha, correlationId) {
await gitOperations.updateRef(githubToken, owner, repo, `heads/${branch}`, sha, true, correlationId);
logger.info('Rollback completed', {
correlationId,
owner,
repo,
branch,
rollbackSha: sha
});
}
/**
* Calculate optimal batch size based on file sizes
* @param {Array} files - Files to process
* @param {number} maxTotalSize - Maximum total size per batch (bytes)
* @param {number} maxFiles - Maximum files per batch
* @returns {Array} Array of batches
*/
calculateBatches(files, maxTotalSize = 5242880, maxFiles = 50) {
const batches = [];
let currentBatch = [];
let currentSize = 0;
for (const file of files) {
const fileSize = Buffer.byteLength(file.content, 'utf8');
if (currentBatch.length >= maxFiles || currentSize + fileSize > maxTotalSize) {
if (currentBatch.length > 0) {
batches.push(currentBatch);
}
currentBatch = [];
currentSize = 0;
}
currentBatch.push(file);
currentSize += fileSize;
}
if (currentBatch.length > 0) {
batches.push(currentBatch);
}
return batches;
}
/**
* Get processing progress
* @param {Object} results - Current processing results
* @returns {Object} Progress info
*/
getProgress(results) {
return {
totalBatches: results.totalBatches,
completedBatches: results.successfulBatches + results.failedBatches,
progress: results.totalBatches > 0
? ((results.successfulBatches + results.failedBatches) / results.totalBatches * 100).toFixed(2)
: 0,
totalFiles: results.totalFiles,
processedFiles: results.processedFiles,
fileProgress: results.totalFiles > 0
? (results.processedFiles / results.totalFiles * 100).toFixed(2)
: 0
};
}
}
// Export singleton instance
export const batchProcessor = new BatchProcessor();
