/** * @fileoverview Audit existing MCP tools against API specifications * Compare each existing MCP tool against corresponding API endpoints to identify * missing parameters, incorrect types, wrong requirement status, and description mismatches. */ import fs from "fs"; import path from "path"; /** * Audit existing MCP tools against API specifications */ function auditMCPTools() { try { console.log("=== AUDITING MCP TOOLS AGAINST API SPECIFICATIONS ==="); console.log(""); // Read the complete API parameters const apiParametersPath = path.join( process.cwd(), "complete-api-parameters.json" ); const apiParametersContent = fs.readFileSync(apiParametersPath, "utf8"); const apiEndpoints = JSON.parse(apiParametersContent); // Read the endpoints configuration const endpointsConfigPath = path.join( process.cwd(), "src/config/endpoints.js" ); const endpointsConfigContent = fs.readFileSync(endpointsConfigPath, "utf8"); // Create a map of API endpoints by path and method const apiEndpointMap = new Map(); apiEndpoints.forEach((endpoint) => { const key = `${endpoint.method}:${endpoint.path}`; apiEndpointMap.set(key, endpoint); }); console.log(`📊 API ENDPOINTS LOADED: ${apiEndpoints.length}`); console.log(`🔍 STARTING TOOL AUDIT...`); console.log(""); const auditResults = { totalApiEndpoints: apiEndpoints.length, totalToolsFound: 0, toolsWithIssues: 0, missingTools: [], toolIssues: [], parameterMismatches: [], authenticationIssues: [], }; // Extract existing tool definitions from endpoints.js const existingTools = extractToolsFromConfig(endpointsConfigContent); auditResults.totalToolsFound = existingTools.length; console.log(`🔧 EXISTING TOOLS FOUND: ${existingTools.length}`); console.log(""); // Audit each API endpoint apiEndpoints.forEach((apiEndpoint) => { const endpointKey = `${apiEndpoint.method}:${apiEndpoint.path}`; // Find corresponding tool const matchingTool = findMatchingTool(existingTools, apiEndpoint); if (!matchingTool) { auditResults.missingTools.push({ path: apiEndpoint.path, method: apiEndpoint.method, summary: apiEndpoint.summary, tags: apiEndpoint.tags, parameterCount: getTotalParameterCount(apiEndpoint), }); } else { // Audit the matching tool const toolAudit = auditTool(matchingTool, apiEndpoint); if (toolAudit.hasIssues) { auditResults.toolsWithIssues++; auditResults.toolIssues.push(toolAudit); } } }); // Generate audit report generateAuditReport(auditResults); // Save detailed audit results const auditOutputPath = path.join( process.cwd(), "mcp-tools-audit-results.json" ); fs.writeFileSync(auditOutputPath, JSON.stringify(auditResults, null, 2)); console.log(`✅ Audit completed. Results saved to: ${auditOutputPath}`); return auditResults; } catch (error) { console.error("Error auditing MCP tools:", error); throw error; } } /** * Extract tool definitions from endpoints configuration */ function extractToolsFromConfig(configContent) { const tools = []; try { // Extract endpoint arrays using regex patterns const endpointSections = [ "PUBLIC_ENDPOINTS", "PROVIDER_ENDPOINTS", "PATIENT_ENDPOINTS", "PARTNER_ENDPOINTS", "AFFILIATE_ENDPOINTS", "NETWORK_ENDPOINTS", ]; endpointSections.forEach((sectionName) => { const authType = sectionName.replace("_ENDPOINTS", "").toLowerCase(); const sectionRegex = new RegExp( `export const ${sectionName}\\s*=\\s*\\[([\\s\\S]*?)\\];`, "g" ); const match = sectionRegex.exec(configContent); if (match) { const sectionContent = match[1]; // Extract individual endpoint objects const endpointRegex = /\{[\s\S]*?\}/g; let endpointMatch; while ((endpointMatch = endpointRegex.exec(sectionContent)) !== null) { const endpointStr = endpointMatch[0]; // Extract path, method, and parameters const pathMatch = endpointStr.match(/path:\s*["']([^"']+)["']/); const methodMatch = endpointStr.match(/method:\s*["']([^"']+)["']/); const descMatch = endpointStr.match( /description:\s*["']([^"']+)["']/ ); if (pathMatch && methodMatch) { const tool = { name: generateToolName(authType, pathMatch[1], methodMatch[1]), authType: authType, path: pathMatch[1], method: methodMatch[1].toUpperCase(), description: descMatch ? descMatch[1] : "", parameters: extractParametersFromEndpoint(endpointStr), }; tools.push(tool); } } } }); } catch (error) { console.warn("Error extracting tools from config:", error.message); } return tools; } /** * Generate tool name from auth type, path, and method */ function generateToolName(authType, path, method) { const action = method.toLowerCase(); const resource = path .split("/") .filter((part) => part && !part.startsWith("{")) .join("_"); return `${authType}_${action}_${resource}` .replace(/[^a-z0-9_]/g, "_") .replace(/_+/g, "_"); } /** * Extract parameters from endpoint string */ function extractParametersFromEndpoint(endpointStr) { const parameters = []; // Look for parameters object const paramMatch = endpointStr.match(/parameters:\s*\{([\s\S]*?)\}/); if (paramMatch) { const paramContent = paramMatch[1]; // Extract individual parameter definitions const paramRegex = /(\w+):\s*\{([^}]+)\}/g; let match; while ((match = paramRegex.exec(paramContent)) !== null) { const paramName = match[1]; const paramDef = match[2]; const typeMatch = paramDef.match(/type:\s*["']([^"']+)["']/); const requiredMatch = paramDef.match(/required:\s*(true|false)/); const descMatch = paramDef.match(/description:\s*["']([^"']+)["']/); parameters.push({ name: paramName, type: typeMatch ? typeMatch[1] : "string", required: requiredMatch ? requiredMatch[1] === "true" : false, description: descMatch ? descMatch[1] : "", }); } } return parameters; } /** * Find matching tool for an API endpoint */ function findMatchingTool(tools, apiEndpoint) { // First try exact path and method match let exactMatch = tools.find( (tool) => tool.path === apiEndpoint.path && tool.method === apiEndpoint.method ); if (exactMatch) return exactMatch; // Try path pattern matching (handle path parameters) const normalizedApiPath = apiEndpoint.path.replace(/\{[^}]+\}/g, "{param}"); let pathMatch = tools.find((tool) => { const normalizedToolPath = tool.path.replace(/\{[^}]+\}/g, "{param}"); return ( normalizedToolPath === normalizedApiPath && tool.method === apiEndpoint.method ); }); if (pathMatch) return pathMatch; // Try matching by operation ID or summary if (apiEndpoint.operationId) { let operationMatch = tools.find( (tool) => tool.name.includes(apiEndpoint.operationId.toLowerCase()) || tool.path.includes(apiEndpoint.operationId.toLowerCase()) ); if (operationMatch) return operationMatch; } return null; } /** * Audit a specific tool against its API endpoint */ function auditTool(tool, apiEndpoint) { const audit = { toolName: tool.name, apiPath: apiEndpoint.path, apiMethod: apiEndpoint.method, hasIssues: false, issues: [], }; // Check parameter completeness const apiParameters = getAllParameters(apiEndpoint); const toolParameters = tool.parameters || []; // Find missing parameters apiParameters.forEach((apiParam) => { const toolParam = toolParameters.find((tp) => tp.name === apiParam.name); if (!toolParam) { audit.hasIssues = true; audit.issues.push({ type: "missing_parameter", parameter: apiParam.name, parameterType: apiParam.type, required: apiParam.required, description: "Parameter exists in API but not in tool", }); } else { // Check parameter type mismatch if (toolParam.type !== apiParam.type) { audit.hasIssues = true; audit.issues.push({ type: "type_mismatch", parameter: apiParam.name, apiType: apiParam.type, toolType: toolParam.type, description: "Parameter type differs between API and tool", }); } // Check required status mismatch if (toolParam.required !== apiParam.required) { audit.hasIssues = true; audit.issues.push({ type: "requirement_mismatch", parameter: apiParam.name, apiRequired: apiParam.required, toolRequired: toolParam.required, description: "Parameter requirement status differs between API and tool", }); } } }); // Find extra parameters in tool toolParameters.forEach((toolParam) => { const apiParam = apiParameters.find((ap) => ap.name === toolParam.name); if (!apiParam) { audit.hasIssues = true; audit.issues.push({ type: "extra_parameter", parameter: toolParam.name, description: "Parameter exists in tool but not in API", }); } }); return audit; } /** * Get all parameters from an API endpoint */ function getAllParameters(apiEndpoint) { const allParams = []; // Add path parameters if (apiEndpoint.parameters?.path) { allParams.push(...apiEndpoint.parameters.path); } // Add query parameters if (apiEndpoint.parameters?.query) { allParams.push(...apiEndpoint.parameters.query); } // Add body parameters if (apiEndpoint.parameters?.body) { allParams.push(...apiEndpoint.parameters.body); } return allParams; } /** * Get total parameter count for an endpoint */ function getTotalParameterCount(apiEndpoint) { return getAllParameters(apiEndpoint).length; } /** * Generate comprehensive audit report */ function generateAuditReport(auditResults) { console.log("=== MCP TOOLS AUDIT REPORT ==="); console.log(""); console.log("📊 OVERVIEW:"); console.log(`Total API endpoints: ${auditResults.totalApiEndpoints}`); console.log(`Total existing tools: ${auditResults.totalToolsFound}`); console.log(`Tools with issues: ${auditResults.toolsWithIssues}`); console.log(`Missing tools: ${auditResults.missingTools.length}`); console.log(""); if (auditResults.missingTools.length > 0) { console.log("❌ MISSING TOOLS:"); auditResults.missingTools.slice(0, 10).forEach((missing) => { console.log(` • ${missing.method} ${missing.path} - ${missing.summary}`); }); if (auditResults.missingTools.length > 10) { console.log(` ... and ${auditResults.missingTools.length - 10} more`); } console.log(""); } if (auditResults.toolIssues.length > 0) { console.log("⚠️ TOOL ISSUES:"); auditResults.toolIssues.slice(0, 5).forEach((issue) => { console.log(` • ${issue.toolName}: ${issue.issues.length} issues`); issue.issues.slice(0, 3).forEach((detail) => { console.log(` - ${detail.type}: ${detail.parameter || "N/A"}`); }); }); if (auditResults.toolIssues.length > 5) { console.log( ` ... and ${auditResults.toolIssues.length - 5} more tools with issues` ); } console.log(""); } // Calculate coverage percentage const coveragePercentage = ( (auditResults.totalToolsFound / auditResults.totalApiEndpoints) * 100 ).toFixed(1); console.log( `📈 COVERAGE: ${coveragePercentage}% of API endpoints have corresponding tools` ); console.log(""); } // Run the audit if (import.meta.url === `file://${process.argv[1]}`) { auditMCPTools(); } export { auditMCPTools };