mcp-tool/verify-all-tools.js

#!/usr/bin/env node

/**
 * Comprehensive Tool Verification Script
 * Verifies all tool names and parameters against the actual endpoint configuration
 */

import fs from 'fs';
import path from 'path';

/**
 * Load and parse endpoints from configuration
 */
function loadEndpointsFromConfig() {
  try {
    const endpointsPath = path.join(process.cwd(), 'src/config/endpoints.js');
    const content = fs.readFileSync(endpointsPath, 'utf8');
    
    const endpoints = {
      PUBLIC: extractEndpointsFromArray(content, 'PUBLIC_ENDPOINTS'),
      PROVIDER: extractEndpointsFromArray(content, 'PROVIDER_ENDPOINTS'),
      PATIENT: extractEndpointsFromArray(content, 'PATIENT_ENDPOINTS'),
      PARTNER: extractEndpointsFromArray(content, 'PARTNER_ENDPOINTS'),
      AFFILIATE: extractEndpointsFromArray(content, 'AFFILIATE_ENDPOINTS'),
      NETWORK: extractEndpointsFromArray(content, 'NETWORK_ENDPOINTS')
    };
    
    return endpoints;
  } catch (error) {
    console.error('❌ Error loading endpoints:', error.message);
    process.exit(1);
  }
}

/**
 * Extract endpoints from a specific array in the configuration
 */
function extractEndpointsFromArray(content, arrayName) {
  const regex = new RegExp(`export const ${arrayName} = \\[([\\s\\S]*?)\\];`);
  const match = content.match(regex);
  
  if (!match) {
    console.warn(`⚠️ Could not find ${arrayName}`);
    return [];
  }
  
  const arrayContent = match[1];
  const endpoints = [];
  
  // Split by endpoint objects (looking for opening braces that start new objects)
  const endpointBlocks = arrayContent.split(/(?=\s*\{[\s\S]*?path:)/);
  
  endpointBlocks.forEach(block => {
    if (!block.trim() || !block.includes('path:')) return;
    
    // Extract path
    const pathMatch = block.match(/path:\s*["']([^"']+)["']/);
    if (!pathMatch) return;
    
    // Extract method
    const methodMatch = block.match(/method:\s*["']([^"']+)["']/);
    if (!methodMatch) return;
    
    // Extract description
    const descMatch = block.match(/description:\s*["']([^"']*?)["']/);
    const description = descMatch ? descMatch[1] : 'No description';
    
    // Extract parameters
    const paramMatch = block.match(/parameters:\s*\{([\s\S]*?)\}(?:\s*,\s*\}|\s*\})/);
    const parameters = paramMatch ? extractParametersFromText(paramMatch[1]) : {};
    
    endpoints.push({
      path: pathMatch[1],
      method: methodMatch[1].toUpperCase(),
      description,
      parameters
    });
  });
  
  return endpoints;
}

/**
 * Extract parameters from parameter block text
 */
function extractParametersFromText(paramText) {
  const parameters = {};
  
  // Match parameter definitions
  const paramRegex = /(\w+):\s*\{[\s\S]*?type:\s*["']([^"']+)["'][\s\S]*?required:\s*(true|false)[\s\S]*?description:\s*["']([^"']*?)["'][\s\S]*?\}/g;
  
  let match;
  while ((match = paramRegex.exec(paramText)) !== null) {
    const [, name, type, required, description] = match;
    parameters[name] = {
      type,
      required: required === 'true',
      description
    };
  }
  
  return parameters;
}

/**
 * Load tools from documentation
 */
function loadToolsFromDocumentation() {
  try {
    const docPath = path.join(process.cwd(), 'MCP-TOOLS-REFERENCE.md');
    const content = fs.readFileSync(docPath, 'utf8');
    
    const tools = {};
    const sections = ['Public', 'Provider', 'Patient', 'Partner', 'Affiliate', 'Network'];
    
    sections.forEach(section => {
      const sectionRegex = new RegExp(`## ${section} Tools[\\s\\S]*?\\| Tool Name[\\s\\S]*?\\n([\\s\\S]*?)(?=\\n##|\\n---|$)`);
      const sectionMatch = content.match(sectionRegex);
      
      if (sectionMatch) {
        const tableContent = sectionMatch[1];
        const toolRegex = /\|\s*`([^`]+)`\s*\|\s*(\w+)\s*\|\s*`([^`]+)`\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|/g;
        
        tools[section.toUpperCase()] = [];
        
        let toolMatch;
        while ((toolMatch = toolRegex.exec(tableContent)) !== null) {
          const [, toolName, method, endpoint, description, parameters] = toolMatch;
          tools[section.toUpperCase()].push({
            toolName: toolName.trim(),
            method: method.trim(),
            endpoint: endpoint.trim(),
            description: description.trim(),
            parameters: parameters.trim()
          });
        }
      }
    });
    
    return tools;
  } catch (error) {
    console.error('❌ Error loading documentation:', error.message);
    process.exit(1);
  }
}

/**
 * Generate tool name from endpoint
 */
function generateToolName(authType, method, path) {
  const pathParts = path
    .replace(/^\/api\//, '')
    .replace(/\{[^}]+\}/g, 'id')
    .replace(/[\/\-]/g, '_')
    .replace(/[^a-zA-Z0-9_]/g, '')
    .toLowerCase();
  
  return `${authType.toLowerCase()}_${method.toLowerCase()}_${pathParts}`;
}

/**
 * Format parameters for comparison
 */
function formatParameters(parameters) {
  if (!parameters || Object.keys(parameters).length === 0) {
    return 'No parameters';
  }
  
  const required = Object.entries(parameters).filter(([, param]) => param.required);
  const optional = Object.entries(parameters).filter(([, param]) => !param.required);
  
  let result = '';
  
  if (required.length > 0) {
    result += '**Required:** ' + required.map(([name, param]) => `${name} (${param.type})`).join(', ');
  }
  
  if (optional.length > 0) {
    if (result) result += ', ';
    result += '**Optional:** ' + optional.map(([name, param]) => `${name} (${param.type})`).join(', ');
  }
  
  return result;
}

/**
 * Verify all tools
 */
function verifyAllTools() {
  console.log('🔍 Starting comprehensive tool verification...\n');
  
  // Load data
  console.log('📋 Loading endpoint configuration...');
  const configEndpoints = loadEndpointsFromConfig();
  
  console.log('📋 Loading documentation tools...');
  const docTools = loadToolsFromDocumentation();
  
  const issues = [];
  let totalConfigEndpoints = 0;
  let totalDocTools = 0;
  let correctTools = 0;
  
  // Verify each authentication type
  Object.keys(configEndpoints).forEach(authType => {
    const endpoints = configEndpoints[authType];
    const tools = docTools[authType] || [];
    
    totalConfigEndpoints += endpoints.length;
    totalDocTools += tools.length;
    
    console.log(`\n🔍 Verifying ${authType} tools...`);
    console.log(`Config endpoints: ${endpoints.length}, Doc tools: ${tools.length}`);
    
    // Check each endpoint
    endpoints.forEach(endpoint => {
      const expectedToolName = generateToolName(authType, endpoint.method, endpoint.path);
      const expectedParameters = formatParameters(endpoint.parameters);
      
      // Find corresponding tool in documentation
      const docTool = tools.find(tool => 
        tool.endpoint === endpoint.path && 
        tool.method === endpoint.method
      );
      
      if (!docTool) {
        issues.push({
          type: 'MISSING_TOOL',
          authType,
          endpoint: endpoint.path,
          method: endpoint.method,
          expectedToolName,
          issue: 'Tool not found in documentation'
        });
      } else {
        // Check tool name
        if (docTool.toolName !== expectedToolName) {
          issues.push({
            type: 'WRONG_TOOL_NAME',
            authType,
            endpoint: endpoint.path,
            expected: expectedToolName,
            actual: docTool.toolName,
            issue: 'Tool name mismatch'
          });
        }
        
        // Check parameters
        if (docTool.parameters !== expectedParameters) {
          issues.push({
            type: 'WRONG_PARAMETERS',
            authType,
            endpoint: endpoint.path,
            toolName: expectedToolName,
            expected: expectedParameters,
            actual: docTool.parameters,
            issue: 'Parameter mismatch'
          });
        } else {
          correctTools++;
        }
      }
    });
  });
  
  // Generate report
  const report = {
    timestamp: new Date().toISOString(),
    summary: {
      totalConfigEndpoints,
      totalDocTools,
      correctTools,
      issues: issues.length,
      accuracy: totalConfigEndpoints > 0 ? ((correctTools / totalConfigEndpoints) * 100).toFixed(1) + '%' : '0%'
    },
    issues,
    byAuthType: {}
  };
  
  // Group issues by auth type
  Object.keys(configEndpoints).forEach(authType => {
    const authIssues = issues.filter(issue => issue.authType === authType);
    report.byAuthType[authType] = {
      endpoints: configEndpoints[authType].length,
      tools: docTools[authType] ? docTools[authType].length : 0,
      issues: authIssues.length
    };
  });
  
  // Save report
  const reportPath = path.join(process.cwd(), 'tool-verification-report.json');
  fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
  
  // Display summary
  console.log('\n📊 VERIFICATION SUMMARY:');
  console.log(`Total config endpoints: ${totalConfigEndpoints}`);
  console.log(`Total doc tools: ${totalDocTools}`);
  console.log(`Correct tools: ${correctTools}`);
  console.log(`Issues found: ${issues.length}`);
  console.log(`Accuracy: ${report.summary.accuracy}`);
  
  if (issues.length > 0) {
    console.log('\n❌ ISSUES FOUND:');
    const issueTypes = {};
    issues.forEach(issue => {
      issueTypes[issue.type] = (issueTypes[issue.type] || 0) + 1;
    });
    
    Object.entries(issueTypes).forEach(([type, count]) => {
      console.log(`  ${type}: ${count} issues`);
    });
    
    console.log('\n🔧 Sample issues:');
    issues.slice(0, 5).forEach(issue => {
      console.log(`  - ${issue.type}: ${issue.endpoint} (${issue.issue})`);
    });
  }
  
  console.log(`\n✅ Report saved to: ${reportPath}`);
  return report;
}

// Run verification if called directly
if (process.argv[1] && process.argv[1].endsWith('verify-all-tools.js')) {
  verifyAllTools();
}

export { verifyAllTools };