GitHub MCP Server

Instructions

Implementation Reference

• src/mcp/tools.ts:631-714 (handler)

  Core handler function implementing github.getCommentImpact. Validates parameters, fetches review comments, groups by PR, fetches PR timelines, analyzes impact using analyzeCommentImpact helper, filters for actual impacts, and returns structured response with stats.

  async getCommentImpact(
    username: string,
    repos: string[],
    from?: string,
    to?: string
  ): Promise<CommentImpactResponse> {
    const { normalizedUsername, normalizedRepos, from: validatedFrom, to: validatedTo } =
      this.validateCommonParameters(username, repos, from, to);
  
    // Get all comments as objects (needed for PR IDs and other metadata)
    const allComments = await this.getReviewCommentsAsObjects(normalizedUsername, validatedFrom, validatedTo);
  
    // Filter comments using shared filter method
    const comments = allComments.filter(comment =>
      this.filterReviewComment(comment, normalizedRepos)
    );
  
    // For each comment, fetch PR timeline and analyze impact
    const impacts: CommentImpact[] = [];
  
    // Group comments by PR to minimize API calls
    const commentsByPR = new Map<string, ReviewComment[]>();
    for (const comment of comments) {
      if (!commentsByPR.has(comment.prId)) {
        commentsByPR.set(comment.prId, []);
      }
      commentsByPR.get(comment.prId)!.push(comment);
    }
  
    // Fetch timeline for each PR and analyze
    for (const [prId, prComments] of commentsByPR.entries()) {
      try {
        const response = await this.client.query(QUERIES.PRTimeline, {
          prId,
        });
  
        const timeline = response.data.node;
  
        if (!timeline) {
          console.error(`[getCommentImpact] No timeline data for PR ${prId}`);
          continue;
        }
  
        // Log commit count for debugging
        const commitCount = timeline.commits?.nodes?.length || 0;
        const timelineCommitCount = (timeline.timelineItems?.nodes || []).filter(
          (item: any) => item.__typename === 'PullRequestCommit'
        ).length;
  
        if (commitCount === 0 && timelineCommitCount === 0) {
          console.error(`[getCommentImpact] No commits found in PR ${prId} timeline`);
        }
  
        for (const comment of prComments) {
          const impact = analyzeCommentImpact(comment, timeline);
          // Only include impacts where there was actual impact (commits found after comment)
          if (impact.hadImpact) {
            impacts.push(impact);
          }
        }
      } catch (error: any) {
        // If timeline fetch fails, skip this PR (don't include in results)
        // Only include impacts where we successfully analyzed and found evidence
        console.error(`Failed to fetch timeline for PR ${prId}:`, error.message);
        // Don't add failed fetches to impacts - only include successful analyses with actual impact
      }
    }
  
    // Calculate statistics from existing data (no additional API calls)
    const response: CommentImpactResponse = {
      impacts,
    };
  
    // Only include stats if we have data to report
    if (comments.length > 0) {
      response.stats = {
        totalComments: comments.length,
        totalPRsReviewed: commentsByPR.size,
        totalImpacts: impacts.length,
      };
    }
  
    return response;
  }

• src/mcp/server.ts:112-127 (registration)

  MCP server request handler registration for 'github.getCommentImpact'. Dispatches to GitHubTools.getCommentImpact and formats response as MCP content.

  case 'github.getCommentImpact': {
    const result = await tools.getCommentImpact(
      args.username as string,
      args.repos as string[],
      args.from as string | undefined,
      args.to as string | undefined
    );
    return {
      content: [
        {
          type: 'text',
          text: JSON.stringify(result, null, 2),
        },
      ],
    };
  }

• src/mcp/tools.ts:1249-1285 (schema)

  Tool schema definition including name, description, and input schema for validation. Note potential mismatch: schema expects 'repo' (string) but implementation uses 'repos' (array).

        name: 'github.getCommentImpact',
        description: `Analyze whether review comments resulted in subsequent code changes. Examines PR timeline data to determine if commits were made after comments were submitted. Returns impact assessment with confidence scores (0.0-1.0) and evidence (e.g., "Commit abc1234 modified files after comment"). Only includes comments with actual impact (commits found after comment). Includes statistics: totalComments, totalPRsReviewed, totalImpacts. Filters by repository. Use this tool to measure the effectiveness of code reviews.
  
  Example use cases:
  - Measure review impact (how often comments lead to code changes)
  - Assess review quality and influence
  - Track review effectiveness metrics
  - Identify high-impact reviewers
  
  Returns: Object with impacts array (commentId, prId, hadImpact, confidence, evidence) and optional stats object (totalComments, totalPRsReviewed, totalImpacts)`,
        inputSchema: {
          type: 'object',
          properties: {
            username: {
              type: 'string',
              description: 'GitHub username (case-insensitive, @ prefix optional). Examples: "octocat", "@octocat"',
              examples: ['octocat', '@octocat'],
            },
            repo: {
              type: 'string',
              description: 'Repository in owner/repo format. Required - only comments for PRs in this repository will be analyzed. Example: "owner/repo"',
              examples: ['owner/repo', 'radireddy/AiApps'],
            },
            from: {
              type: 'string',
              description: 'Start timestamp in ISO 8601 format. Example: "2024-01-01T00:00:00Z"',
              examples: ['2024-01-01T00:00:00Z'],
            },
            to: {
              type: 'string',
              description: 'End timestamp in ISO 8601 format. Example: "2024-12-31T23:59:59Z"',
              examples: ['2024-12-31T23:59:59Z'],
            },
          },
          required: ['username', 'repo', 'from', 'to'],
        },
      },

• src/github/mapper.ts:257-359 (helper)

  Key helper function that analyzes individual comment impact by checking for commits in PR timeline after comment timestamp. Computes confidence and evidence, central to the tool's logic.

  export function analyzeCommentImpact(
    comment: ReviewComment,
    prTimeline: any
  ): CommentImpact {
    const evidence: string[] = [];
    let hadImpact = false;
    let confidence = 0;
  
    if (!prTimeline?.timelineItems?.nodes) {
      // Return hadImpact: false with empty evidence - caller will filter these out
      return {
        commentId: comment.id,
        prId: comment.prId,
        hadImpact: false,
        confidence: 0,
        evidence: [],
      };
    }
  
    const commentTime = new Date(comment.timestamp).getTime();
    
    // Get commits from PR commits (more reliable than timeline items)
    // Commits are ordered chronologically, so we can check if any came after the comment
    const commits = prTimeline.commits?.nodes || [];
    
    // Also check timeline items for commits (fallback)
    const timelineCommits = (prTimeline.timelineItems?.nodes || []).filter(
      (item: any) => item.__typename === 'PullRequestCommit'
    );
  
    // Combine both sources and deduplicate by commit ID
    const allCommits = new Map<string, any>();
    
    for (const commitNode of commits) {
      if (commitNode?.commit?.id) {
        allCommits.set(commitNode.commit.id, commitNode.commit);
      }
    }
    
    for (const commitItem of timelineCommits) {
      if (commitItem?.commit?.id) {
        allCommits.set(commitItem.commit.id, commitItem.commit);
      }
    }
  
    for (const commit of allCommits.values()) {
      // Use committedDate if available (when commit was actually committed/pushed to branch)
      // Otherwise fall back to authoredDate (when commit was originally authored)
      // Note: authoredDate can be earlier than when commit was pushed to PR, so we prefer committedDate
      const commitDateStr = commit.committedDate || commit.authoredDate;
      if (!commitDateStr) continue;
      
      const commitTime = new Date(commitDateStr).getTime();
      
      // Skip commits that were made before or at the same time as the comment
      // We want commits that happened AFTER the comment to show impact
      // Using <= to skip commits at the same timestamp (allowing for small timing differences)
      if (commitTime <= commentTime) {
        continue;
      }
      
      // Found a commit after the comment - this indicates potential impact
  
      // If comment is on a specific file/line, check if that file was modified
      if (comment.filePath) {
        // Check if file was in commit changes
        // Note: GitHub GraphQL doesn't expose file-level commit details easily
        // This is a simplified heuristic - higher confidence if files were changed
        if (commit.changedFiles > 0) {
          evidence.push(
            `Commit ${commit.id.substring(0, 7)} modified files after comment (${commit.changedFiles} files)`
          );
          hadImpact = true;
          confidence = 0.5; // Medium confidence without file-level detail
        }
      } else {
        // General comment - lower confidence
        if (commit.changedFiles > 0) {
          evidence.push(
            `General comment followed by commit ${commit.id.substring(0, 7)}`
          );
          hadImpact = true;
          confidence = 0.3;
        }
      }
    }
  
    // Higher confidence if comment requested changes (would need additional query to verify)
    if (hadImpact && comment.reviewId) {
      confidence = Math.min(confidence + 0.2, 1.0);
    }
  
    // Only return impact if there was actual impact (commits found after comment)
    // Don't include "No commits found" message - just return hadImpact: false
    // The caller will filter these out
    return {
      commentId: comment.id,
      prId: comment.prId,
      hadImpact,
      confidence,
      evidence,
    };
  }