MCP Web Research Server

Instructions

Implementation Reference

• index.ts:991-1153 (handler)

  Core handler implementation for 'search_scholar' tool. Uses Playwright to stealthily navigate Google Scholar, input query with human-like delays and typing, submit search, extract structured results (title, authors, snippet, citations), store in session, and return JSON.

  case "search_scholar": {
      // Extract search query from request parameters
      const { query } = request.params.arguments as { query: string };
  
      try {
          // Execute search with retry mechanism
          const results = await withRetry(async () => {
              // Step 1: Navigate to Google Scholar search page
              await safePageNavigation(page, 'https://scholar.google.com');
  
              // Simulate human behavior
              await randomDelay(800, 1500);
              await simulateHumanBehavior(page);
  
              // Step 2: Find and interact with search input
              await withRetry(async () => {
                  // Wait for search input element to appear
                  await page.waitForSelector('input[name="q"]', { timeout: 5000 })
                  .catch(() => {
                      throw new Error('Scholar search input not found');
                  });
  
                  // Random delay before interacting
                  await randomDelay(300, 700);
  
                  // Find the search input element
                  const searchInput = await page.$('input[name="q"]');
  
                  // Verify search input was found
                  if (!searchInput) {
                      throw new Error('Scholar search input element not found after waiting');
                  }
  
                  // Step 3: Enter search query with human-like typing
                  await searchInput.click();
                  await randomDelay(100, 300);
                  await searchInput.click({ clickCount: 3 });  // Select all existing text
                  await randomDelay(50, 150);
                  await searchInput.press('Backspace');        // Clear selected text
                  await randomDelay(200, 400);
  
                  // Type query with random delays between characters
                  for (const char of query) {
                      await searchInput.type(char);
                      await randomDelay(50, 150);
                  }
              }, 3, 2000);  // Allow 3 retries with 2s delay
  
              // Random delay before submitting
              await randomDelay(400, 900);
  
              // Step 4: Submit search and wait for results
              await withRetry(async () => {
                  await Promise.all([
                      page.keyboard.press('Enter'),
                                    page.waitForLoadState('networkidle', { timeout: 15000 }),
                  ]);
              });
  
              // Simulate human behavior after results load
              await randomDelay(500, 1000);
              await simulateHumanBehavior(page);
  
              // Step 5: Extract scholar search results
              const scholarResults = await withRetry(async () => {
                  const results = await page.evaluate(() => {
                      // Find all scholar result containers
                      const elements = document.querySelectorAll('.gs_r.gs_or.gs_scl');
                      if (!elements || elements.length === 0) {
                          throw new Error('No scholar search results found');
                      }
  
                      // Extract data from each result
                      return Array.from(elements).map((el) => {
                          try {
                              // Find required elements within result container
                              const titleEl = el.querySelector('.gs_rt');                // Title element
                              const authorEl = el.querySelector('.gs_a');                // Authors, venue, year
                              const snippetEl = el.querySelector('.gs_rs');              // Snippet/abstract
                              const citedByEl = el.querySelector('.gs_fl a:nth-child(3)'); // Cited by element
  
                              // Extract title and URL
                              let title = '';
                              let url = '';
                      if (titleEl) {
                          const titleLink = titleEl.querySelector('a');
                          title = titleEl.textContent?.trim() || '';
                          url = titleLink?.getAttribute('href') || '';
                      }
  
                      // Extract author, venue, and year information
                      const authorInfo = authorEl?.textContent?.trim() || '';
  
                      // Extract snippet
                      const snippet = snippetEl?.textContent?.trim() || '';
  
                      // Extract citation count
                      let citationCount = '';
                      if (citedByEl && citedByEl.textContent?.includes('Cited by')) {
                          citationCount = citedByEl.textContent.trim();
                      }
  
                      // Skip results missing critical data
                      if (!title) {
                          return null;
                      }
  
                      // Return structured result data
                      return {
                          title,                 // Paper title
                          url,                   // Paper URL if available
                          authorInfo,            // Authors, venue, year
                          snippet,               // Abstract/snippet
                          citationCount,         // Citation information
                      };
                          } catch (err) {
                              // Skip problematic results
                              return null;
                          }
                      }).filter(result => result !== null);  // Remove invalid results
                  });
  
                  // Verify we found valid results
                  if (!results || results.length === 0) {
                      throw new Error('No valid scholar search results found');
                  }
  
                  // Return compiled list of results
                  return results;
              });
  
              // Step 6: Store results in session
              scholarResults.forEach((result) => {
                  addResult({
                      url: result.url || 'https://scholar.google.com',
                      title: result.title,
                      content: `${result.authorInfo}\n\n${result.snippet}\n\n${result.citationCount}`,
                      timestamp: new Date().toISOString(),
                  });
              });
  
              // Return compiled list of results
              return scholarResults;
          });
  
          // Step 7: Return formatted results
          return {
              content: [{
                  type: "text",
                  text: JSON.stringify(results, null, 2)  // Pretty-print JSON results
              }]
          };
      } catch (error) {
          // Handle and format search errors
          return {
              content: [{
                  type: "text",
                  text: `Failed to perform scholar search: ${(error as Error).message}`
              }],
              isError: true
          };
      }
  }

• index.ts:178-184 (schema)

  Input schema definition for search_scholar tool, requiring a 'query' string parameter.

  inputSchema: {
      type: "object",
      properties: {
          query: { type: "string", description: "Academic search query" },
      },
      required: ["query"],
  },

• index.ts:175-185 (registration)

  Tool registration in TOOLS array, defining name, description, and linking to input schema for list tools endpoint.

  {
      name: "search_scholar",
      description: "Searches Google Scholar for academic papers and scholarly articles. Use this tool when researching scientific topics, looking for peer-reviewed research, academic citations, or scholarly literature. Returns structured data including titles, authors, publication details, and citation counts. Ideal for academic research and evidence-based inquiries.",
      inputSchema: {
          type: "object",
          properties: {
              query: { type: "string", description: "Academic search query" },
          },
          required: ["query"],
      },
  },

• index.ts:535-537 (registration)

  Registers the ListToolsRequestSchema handler which returns the TOOLS array including search_scholar.

  server.setRequestHandler(ListToolsRequestSchema, async () => ({
      tools: TOOLS  // Return list of available research tools
  }));