#!/usr/bin/env node /** * Comprehensive Tool Verification Script * Verifies all tool names and parameters against the actual endpoint configuration */ import fs from 'fs'; import path from 'path'; /** * Load and parse endpoints from configuration */ function loadEndpointsFromConfig() { try { const endpointsPath = path.join(process.cwd(), 'src/config/endpoints.js'); const content = fs.readFileSync(endpointsPath, 'utf8'); const endpoints = { PUBLIC: extractEndpointsFromArray(content, 'PUBLIC_ENDPOINTS'), PROVIDER: extractEndpointsFromArray(content, 'PROVIDER_ENDPOINTS'), PATIENT: extractEndpointsFromArray(content, 'PATIENT_ENDPOINTS'), PARTNER: extractEndpointsFromArray(content, 'PARTNER_ENDPOINTS'), AFFILIATE: extractEndpointsFromArray(content, 'AFFILIATE_ENDPOINTS'), NETWORK: extractEndpointsFromArray(content, 'NETWORK_ENDPOINTS') }; return endpoints; } catch (error) { console.error('āŒ Error loading endpoints:', error.message); process.exit(1); } } /** * Extract endpoints from a specific array in the configuration */ function extractEndpointsFromArray(content, arrayName) { const regex = new RegExp(`export const ${arrayName} = \\[([\\s\\S]*?)\\];`); const match = content.match(regex); if (!match) { console.warn(`āš ļø Could not find ${arrayName}`); return []; } const arrayContent = match[1]; const endpoints = []; // Split by endpoint objects (looking for opening braces that start new objects) const endpointBlocks = arrayContent.split(/(?=\s*\{[\s\S]*?path:)/); endpointBlocks.forEach(block => { if (!block.trim() || !block.includes('path:')) return; // Extract path const pathMatch = block.match(/path:\s*["']([^"']+)["']/); if (!pathMatch) return; // Extract method const methodMatch = block.match(/method:\s*["']([^"']+)["']/); if (!methodMatch) return; // Extract description const descMatch = block.match(/description:\s*["']([^"']*?)["']/); const description = descMatch ? descMatch[1] : 'No description'; // Extract parameters const paramMatch = block.match(/parameters:\s*\{([\s\S]*?)\}(?:\s*,\s*\}|\s*\})/); const parameters = paramMatch ? extractParametersFromText(paramMatch[1]) : {}; endpoints.push({ path: pathMatch[1], method: methodMatch[1].toUpperCase(), description, parameters }); }); return endpoints; } /** * Extract parameters from parameter block text */ function extractParametersFromText(paramText) { const parameters = {}; // Match parameter definitions const paramRegex = /(\w+):\s*\{[\s\S]*?type:\s*["']([^"']+)["'][\s\S]*?required:\s*(true|false)[\s\S]*?description:\s*["']([^"']*?)["'][\s\S]*?\}/g; let match; while ((match = paramRegex.exec(paramText)) !== null) { const [, name, type, required, description] = match; parameters[name] = { type, required: required === 'true', description }; } return parameters; } /** * Load tools from documentation */ function loadToolsFromDocumentation() { try { const docPath = path.join(process.cwd(), 'MCP-TOOLS-REFERENCE.md'); const content = fs.readFileSync(docPath, 'utf8'); const tools = {}; const sections = ['Public', 'Provider', 'Patient', 'Partner', 'Affiliate', 'Network']; sections.forEach(section => { const sectionRegex = new RegExp(`## ${section} Tools[\\s\\S]*?\\| Tool Name[\\s\\S]*?\\n([\\s\\S]*?)(?=\\n##|\\n---|$)`); const sectionMatch = content.match(sectionRegex); if (sectionMatch) { const tableContent = sectionMatch[1]; const toolRegex = /\|\s*`([^`]+)`\s*\|\s*(\w+)\s*\|\s*`([^`]+)`\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|/g; tools[section.toUpperCase()] = []; let toolMatch; while ((toolMatch = toolRegex.exec(tableContent)) !== null) { const [, toolName, method, endpoint, description, parameters] = toolMatch; tools[section.toUpperCase()].push({ toolName: toolName.trim(), method: method.trim(), endpoint: endpoint.trim(), description: description.trim(), parameters: parameters.trim() }); } } }); return tools; } catch (error) { console.error('āŒ Error loading documentation:', error.message); process.exit(1); } } /** * Generate tool name from endpoint */ function generateToolName(authType, method, path) { const pathParts = path .replace(/^\/api\//, '') .replace(/\{[^}]+\}/g, 'id') .replace(/[\/\-]/g, '_') .replace(/[^a-zA-Z0-9_]/g, '') .toLowerCase(); return `${authType.toLowerCase()}_${method.toLowerCase()}_${pathParts}`; } /** * Format parameters for comparison */ function formatParameters(parameters) { if (!parameters || Object.keys(parameters).length === 0) { return 'No parameters'; } const required = Object.entries(parameters).filter(([, param]) => param.required); const optional = Object.entries(parameters).filter(([, param]) => !param.required); let result = ''; if (required.length > 0) { result += '**Required:** ' + required.map(([name, param]) => `${name} (${param.type})`).join(', '); } if (optional.length > 0) { if (result) result += ', '; result += '**Optional:** ' + optional.map(([name, param]) => `${name} (${param.type})`).join(', '); } return result; } /** * Verify all tools */ function verifyAllTools() { console.log('šŸ” Starting comprehensive tool verification...\n'); // Load data console.log('šŸ“‹ Loading endpoint configuration...'); const configEndpoints = loadEndpointsFromConfig(); console.log('šŸ“‹ Loading documentation tools...'); const docTools = loadToolsFromDocumentation(); const issues = []; let totalConfigEndpoints = 0; let totalDocTools = 0; let correctTools = 0; // Verify each authentication type Object.keys(configEndpoints).forEach(authType => { const endpoints = configEndpoints[authType]; const tools = docTools[authType] || []; totalConfigEndpoints += endpoints.length; totalDocTools += tools.length; console.log(`\nšŸ” Verifying ${authType} tools...`); console.log(`Config endpoints: ${endpoints.length}, Doc tools: ${tools.length}`); // Check each endpoint endpoints.forEach(endpoint => { const expectedToolName = generateToolName(authType, endpoint.method, endpoint.path); const expectedParameters = formatParameters(endpoint.parameters); // Find corresponding tool in documentation const docTool = tools.find(tool => tool.endpoint === endpoint.path && tool.method === endpoint.method ); if (!docTool) { issues.push({ type: 'MISSING_TOOL', authType, endpoint: endpoint.path, method: endpoint.method, expectedToolName, issue: 'Tool not found in documentation' }); } else { // Check tool name if (docTool.toolName !== expectedToolName) { issues.push({ type: 'WRONG_TOOL_NAME', authType, endpoint: endpoint.path, expected: expectedToolName, actual: docTool.toolName, issue: 'Tool name mismatch' }); } // Check parameters if (docTool.parameters !== expectedParameters) { issues.push({ type: 'WRONG_PARAMETERS', authType, endpoint: endpoint.path, toolName: expectedToolName, expected: expectedParameters, actual: docTool.parameters, issue: 'Parameter mismatch' }); } else { correctTools++; } } }); }); // Generate report const report = { timestamp: new Date().toISOString(), summary: { totalConfigEndpoints, totalDocTools, correctTools, issues: issues.length, accuracy: totalConfigEndpoints > 0 ? ((correctTools / totalConfigEndpoints) * 100).toFixed(1) + '%' : '0%' }, issues, byAuthType: {} }; // Group issues by auth type Object.keys(configEndpoints).forEach(authType => { const authIssues = issues.filter(issue => issue.authType === authType); report.byAuthType[authType] = { endpoints: configEndpoints[authType].length, tools: docTools[authType] ? docTools[authType].length : 0, issues: authIssues.length }; }); // Save report const reportPath = path.join(process.cwd(), 'tool-verification-report.json'); fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); // Display summary console.log('\nšŸ“Š VERIFICATION SUMMARY:'); console.log(`Total config endpoints: ${totalConfigEndpoints}`); console.log(`Total doc tools: ${totalDocTools}`); console.log(`Correct tools: ${correctTools}`); console.log(`Issues found: ${issues.length}`); console.log(`Accuracy: ${report.summary.accuracy}`); if (issues.length > 0) { console.log('\nāŒ ISSUES FOUND:'); const issueTypes = {}; issues.forEach(issue => { issueTypes[issue.type] = (issueTypes[issue.type] || 0) + 1; }); Object.entries(issueTypes).forEach(([type, count]) => { console.log(` ${type}: ${count} issues`); }); console.log('\nšŸ”§ Sample issues:'); issues.slice(0, 5).forEach(issue => { console.log(` - ${issue.type}: ${issue.endpoint} (${issue.issue})`); }); } console.log(`\nāœ… Report saved to: ${reportPath}`); return report; } // Run verification if called directly if (process.argv[1] && process.argv[1].endsWith('verify-all-tools.js')) { verifyAllTools(); } export { verifyAllTools };