mcp-tool/test-final-validation.js

#!/usr/bin/env node

/**
 * Final Validation Test
 * Verify the updated MCP-TOOLS-REFERENCE.md is accurate and complete
 */

import fs from 'fs';
import { ConfigManager } from './src/config/ConfigManager.js';
import { AuthManager } from './src/auth/AuthManager.js';
import { ApiClient } from './src/proxy/ApiClient.js';
import { ToolGenerator } from './src/tools/ToolGenerator.js';

async function finalValidationTest() {
  try {
    console.log('🧪 Final Validation Test for MCP-TOOLS-REFERENCE.md\n');

    // Initialize components
    const config = new ConfigManager();
    const authManager = new AuthManager(null, config.getAll(true));
    const apiClient = new ApiClient(config.getAll(), authManager);
    const toolGenerator = new ToolGenerator(apiClient);

    // Generate all tools
    console.log('📋 Generating all tools from ToolGenerator...');
    const allTools = toolGenerator.generateAllTools();
    console.log(`✅ Generated ${allTools.length} tools from ToolGenerator`);

    // Create unique tools map
    const uniqueTools = new Map();
    allTools.forEach(tool => {
      if (!uniqueTools.has(tool.name)) {
        uniqueTools.set(tool.name, tool);
      }
    });

    console.log(`📊 Unique tools: ${uniqueTools.size}`);
    console.log(`📊 Duplicates: ${allTools.length - uniqueTools.size}`);

    // Read the documentation file
    console.log('\n📖 Reading MCP-TOOLS-REFERENCE.md...');
    const docContent = fs.readFileSync('MCP-TOOLS-REFERENCE.md', 'utf8');
    
    // Extract tool names from documentation
    const toolNameRegex = /### `([^`]+)`/g;
    const docToolNames = [];
    let match;
    while ((match = toolNameRegex.exec(docContent)) !== null) {
      docToolNames.push(match[1]);
    }

    console.log(`📄 Tools documented: ${docToolNames.length}`);

    // Cross-validation
    console.log('\n🔍 Cross-validation between ToolGenerator and Documentation...');
    
    const generatedToolNames = Array.from(uniqueTools.keys()).sort();
    const documentedToolNames = [...new Set(docToolNames)].sort(); // Remove duplicates and sort
    
    console.log(`📊 Generated tools: ${generatedToolNames.length}`);
    console.log(`📊 Documented tools: ${documentedToolNames.length}`);

    // Find missing tools in documentation
    const missingInDoc = generatedToolNames.filter(name => !documentedToolNames.includes(name));
    const extraInDoc = documentedToolNames.filter(name => !generatedToolNames.includes(name));

    if (missingInDoc.length > 0) {
      console.log(`\n⚠️  Tools missing from documentation (${missingInDoc.length}):`);
      missingInDoc.slice(0, 10).forEach(name => console.log(`  - ${name}`));
      if (missingInDoc.length > 10) {
        console.log(`  ... and ${missingInDoc.length - 10} more`);
      }
    }

    if (extraInDoc.length > 0) {
      console.log(`\n⚠️  Extra tools in documentation (${extraInDoc.length}):`);
      extraInDoc.slice(0, 10).forEach(name => console.log(`  - ${name}`));
      if (extraInDoc.length > 10) {
        console.log(`  ... and ${extraInDoc.length - 10} more`);
      }
    }

    // Test specific tools
    console.log('\n🧪 Testing specific tool implementations...');
    
    const testTools = [
      'public_create_login',
      'provider_get_emrpatientsList',
      'provider_create_startCall',
      'patient_get_frontendpatientDashboard'
    ];

    let testsPassed = 0;
    for (const toolName of testTools) {
      const tool = toolGenerator.getTool(toolName);
      if (tool && tool.execute) {
        console.log(`✅ ${toolName}: Found and executable`);
        testsPassed++;
      } else {
        console.log(`❌ ${toolName}: Missing or not executable`);
      }
    }

    // Validate documentation structure
    console.log('\n📋 Validating documentation structure...');
    
    const hasOverview = docContent.includes('## 📊 Overview');
    const hasDistribution = docContent.includes('## 📋 Tool Distribution by Authentication Type');
    const hasPublicSection = docContent.includes('## 🌐 Public Tools');
    const hasProviderSection = docContent.includes('## 🏥 Provider Tools');
    const hasUsageGuidelines = docContent.includes('## 📚 Usage Guidelines');
    const hasSecurityNotes = docContent.includes('## 🔒 Security Notes');

    console.log(`✅ Overview section: ${hasOverview ? 'Present' : 'Missing'}`);
    console.log(`✅ Distribution table: ${hasDistribution ? 'Present' : 'Missing'}`);
    console.log(`✅ Public tools section: ${hasPublicSection ? 'Present' : 'Missing'}`);
    console.log(`✅ Provider tools section: ${hasProviderSection ? 'Present' : 'Missing'}`);
    console.log(`✅ Usage guidelines: ${hasUsageGuidelines ? 'Present' : 'Missing'}`);
    console.log(`✅ Security notes: ${hasSecurityNotes ? 'Present' : 'Missing'}`);

    // Calculate accuracy
    const accuracy = documentedToolNames.length === generatedToolNames.length && 
                    missingInDoc.length === 0 && 
                    extraInDoc.length === 0;

    console.log('\n📊 Final Results:');
    console.log(`🎯 Documentation Accuracy: ${accuracy ? '100%' : 'Needs improvement'}`);
    console.log(`🧪 Tool Tests Passed: ${testsPassed}/${testTools.length}`);
    console.log(`📄 Total Tools Documented: ${documentedToolNames.length}`);
    console.log(`🔧 Total Tools Generated: ${generatedToolNames.length}`);
    console.log(`📊 Coverage: ${((documentedToolNames.length / generatedToolNames.length) * 100).toFixed(1)}%`);

    // Summary by auth type
    console.log('\n📋 Tools by Authentication Type:');
    const authTypes = ['public', 'provider', 'patient', 'partner', 'affiliate', 'network'];
    authTypes.forEach(authType => {
      const authTools = generatedToolNames.filter(name => name.startsWith(authType + '_'));
      const authPercentage = ((authTools.length / generatedToolNames.length) * 100).toFixed(1);
      console.log(`  ${authType.toUpperCase()}: ${authTools.length} tools (${authPercentage}%)`);
    });

    if (accuracy && testsPassed === testTools.length) {
      console.log('\n🎉 All validation tests passed! MCP-TOOLS-REFERENCE.md is accurate and complete.');
      return true;
    } else {
      console.log('\n⚠️  Some validation issues found. Please review the results above.');
      return false;
    }

  } catch (error) {
    console.error('❌ Validation test failed:', error.message);
    console.error('📋 Stack:', error.stack);
    return false;
  }
}

// Run the validation test
finalValidationTest().then(success => {
  if (success) {
    console.log('\n✅ Final validation completed successfully!');
    process.exit(0);
  } else {
    console.log('\n❌ Final validation failed!');
    process.exit(1);
  }
});