mcp-tool/audit-mcp-tools.js

/**
 * @fileoverview Audit existing MCP tools against API specifications
 * Compare each existing MCP tool against corresponding API endpoints to identify
 * missing parameters, incorrect types, wrong requirement status, and description mismatches.
 */

import fs from "fs";
import path from "path";

/**
 * Audit existing MCP tools against API specifications
 */
function auditMCPTools() {
  try {
    console.log("=== AUDITING MCP TOOLS AGAINST API SPECIFICATIONS ===");
    console.log("");

    // Read the complete API parameters
    const apiParametersPath = path.join(
      process.cwd(),
      "complete-api-parameters.json"
    );
    const apiParametersContent = fs.readFileSync(apiParametersPath, "utf8");
    const apiEndpoints = JSON.parse(apiParametersContent);

    // Read the endpoints configuration
    const endpointsConfigPath = path.join(
      process.cwd(),
      "src/config/endpoints.js"
    );
    const endpointsConfigContent = fs.readFileSync(endpointsConfigPath, "utf8");

    // Create a map of API endpoints by path and method
    const apiEndpointMap = new Map();
    apiEndpoints.forEach((endpoint) => {
      const key = `${endpoint.method}:${endpoint.path}`;
      apiEndpointMap.set(key, endpoint);
    });

    console.log(`📊 API ENDPOINTS LOADED: ${apiEndpoints.length}`);
    console.log(`🔍 STARTING TOOL AUDIT...`);
    console.log("");

    const auditResults = {
      totalApiEndpoints: apiEndpoints.length,
      totalToolsFound: 0,
      toolsWithIssues: 0,
      missingTools: [],
      toolIssues: [],
      parameterMismatches: [],
      authenticationIssues: [],
    };

    // Extract existing tool definitions from endpoints.js
    const existingTools = extractToolsFromConfig(endpointsConfigContent);
    auditResults.totalToolsFound = existingTools.length;

    console.log(`🔧 EXISTING TOOLS FOUND: ${existingTools.length}`);
    console.log("");

    // Audit each API endpoint
    apiEndpoints.forEach((apiEndpoint) => {
      const endpointKey = `${apiEndpoint.method}:${apiEndpoint.path}`;

      // Find corresponding tool
      const matchingTool = findMatchingTool(existingTools, apiEndpoint);

      if (!matchingTool) {
        auditResults.missingTools.push({
          path: apiEndpoint.path,
          method: apiEndpoint.method,
          summary: apiEndpoint.summary,
          tags: apiEndpoint.tags,
          parameterCount: getTotalParameterCount(apiEndpoint),
        });
      } else {
        // Audit the matching tool
        const toolAudit = auditTool(matchingTool, apiEndpoint);
        if (toolAudit.hasIssues) {
          auditResults.toolsWithIssues++;
          auditResults.toolIssues.push(toolAudit);
        }
      }
    });

    // Generate audit report
    generateAuditReport(auditResults);

    // Save detailed audit results
    const auditOutputPath = path.join(
      process.cwd(),
      "mcp-tools-audit-results.json"
    );
    fs.writeFileSync(auditOutputPath, JSON.stringify(auditResults, null, 2));

    console.log(`✅ Audit completed. Results saved to: ${auditOutputPath}`);

    return auditResults;
  } catch (error) {
    console.error("Error auditing MCP tools:", error);
    throw error;
  }
}

/**
 * Extract tool definitions from endpoints configuration
 */
function extractToolsFromConfig(configContent) {
  const tools = [];

  try {
    // Extract endpoint arrays using regex patterns
    const endpointSections = [
      "PUBLIC_ENDPOINTS",
      "PROVIDER_ENDPOINTS",
      "PATIENT_ENDPOINTS",
      "PARTNER_ENDPOINTS",
      "AFFILIATE_ENDPOINTS",
      "NETWORK_ENDPOINTS",
    ];

    endpointSections.forEach((sectionName) => {
      const authType = sectionName.replace("_ENDPOINTS", "").toLowerCase();
      const sectionRegex = new RegExp(
        `export const ${sectionName}\\s*=\\s*\\[([\\s\\S]*?)\\];`,
        "g"
      );
      const match = sectionRegex.exec(configContent);

      if (match) {
        const sectionContent = match[1];

        // Extract individual endpoint objects
        const endpointRegex = /\{[\s\S]*?\}/g;
        let endpointMatch;

        while ((endpointMatch = endpointRegex.exec(sectionContent)) !== null) {
          const endpointStr = endpointMatch[0];

          // Extract path, method, and parameters
          const pathMatch = endpointStr.match(/path:\s*["']([^"']+)["']/);
          const methodMatch = endpointStr.match(/method:\s*["']([^"']+)["']/);
          const descMatch = endpointStr.match(
            /description:\s*["']([^"']+)["']/
          );

          if (pathMatch && methodMatch) {
            const tool = {
              name: generateToolName(authType, pathMatch[1], methodMatch[1]),
              authType: authType,
              path: pathMatch[1],
              method: methodMatch[1].toUpperCase(),
              description: descMatch ? descMatch[1] : "",
              parameters: extractParametersFromEndpoint(endpointStr),
            };

            tools.push(tool);
          }
        }
      }
    });
  } catch (error) {
    console.warn("Error extracting tools from config:", error.message);
  }

  return tools;
}

/**
 * Generate tool name from auth type, path, and method
 */
function generateToolName(authType, path, method) {
  const action = method.toLowerCase();
  const resource = path
    .split("/")
    .filter((part) => part && !part.startsWith("{"))
    .join("_");
  return `${authType}_${action}_${resource}`
    .replace(/[^a-z0-9_]/g, "_")
    .replace(/_+/g, "_");
}

/**
 * Extract parameters from endpoint string
 */
function extractParametersFromEndpoint(endpointStr) {
  const parameters = [];

  // Look for parameters object
  const paramMatch = endpointStr.match(/parameters:\s*\{([\s\S]*?)\}/);
  if (paramMatch) {
    const paramContent = paramMatch[1];

    // Extract individual parameter definitions
    const paramRegex = /(\w+):\s*\{([^}]+)\}/g;
    let match;

    while ((match = paramRegex.exec(paramContent)) !== null) {
      const paramName = match[1];
      const paramDef = match[2];

      const typeMatch = paramDef.match(/type:\s*["']([^"']+)["']/);
      const requiredMatch = paramDef.match(/required:\s*(true|false)/);
      const descMatch = paramDef.match(/description:\s*["']([^"']+)["']/);

      parameters.push({
        name: paramName,
        type: typeMatch ? typeMatch[1] : "string",
        required: requiredMatch ? requiredMatch[1] === "true" : false,
        description: descMatch ? descMatch[1] : "",
      });
    }
  }

  return parameters;
}

/**
 * Find matching tool for an API endpoint
 */
function findMatchingTool(tools, apiEndpoint) {
  // First try exact path and method match
  let exactMatch = tools.find(
    (tool) =>
      tool.path === apiEndpoint.path && tool.method === apiEndpoint.method
  );

  if (exactMatch) return exactMatch;

  // Try path pattern matching (handle path parameters)
  const normalizedApiPath = apiEndpoint.path.replace(/\{[^}]+\}/g, "{param}");
  let pathMatch = tools.find((tool) => {
    const normalizedToolPath = tool.path.replace(/\{[^}]+\}/g, "{param}");
    return (
      normalizedToolPath === normalizedApiPath &&
      tool.method === apiEndpoint.method
    );
  });

  if (pathMatch) return pathMatch;

  // Try matching by operation ID or summary
  if (apiEndpoint.operationId) {
    let operationMatch = tools.find(
      (tool) =>
        tool.name.includes(apiEndpoint.operationId.toLowerCase()) ||
        tool.path.includes(apiEndpoint.operationId.toLowerCase())
    );
    if (operationMatch) return operationMatch;
  }

  return null;
}

/**
 * Audit a specific tool against its API endpoint
 */
function auditTool(tool, apiEndpoint) {
  const audit = {
    toolName: tool.name,
    apiPath: apiEndpoint.path,
    apiMethod: apiEndpoint.method,
    hasIssues: false,
    issues: [],
  };

  // Check parameter completeness
  const apiParameters = getAllParameters(apiEndpoint);
  const toolParameters = tool.parameters || [];

  // Find missing parameters
  apiParameters.forEach((apiParam) => {
    const toolParam = toolParameters.find((tp) => tp.name === apiParam.name);
    if (!toolParam) {
      audit.hasIssues = true;
      audit.issues.push({
        type: "missing_parameter",
        parameter: apiParam.name,
        parameterType: apiParam.type,
        required: apiParam.required,
        description: "Parameter exists in API but not in tool",
      });
    } else {
      // Check parameter type mismatch
      if (toolParam.type !== apiParam.type) {
        audit.hasIssues = true;
        audit.issues.push({
          type: "type_mismatch",
          parameter: apiParam.name,
          apiType: apiParam.type,
          toolType: toolParam.type,
          description: "Parameter type differs between API and tool",
        });
      }

      // Check required status mismatch
      if (toolParam.required !== apiParam.required) {
        audit.hasIssues = true;
        audit.issues.push({
          type: "requirement_mismatch",
          parameter: apiParam.name,
          apiRequired: apiParam.required,
          toolRequired: toolParam.required,
          description:
            "Parameter requirement status differs between API and tool",
        });
      }
    }
  });

  // Find extra parameters in tool
  toolParameters.forEach((toolParam) => {
    const apiParam = apiParameters.find((ap) => ap.name === toolParam.name);
    if (!apiParam) {
      audit.hasIssues = true;
      audit.issues.push({
        type: "extra_parameter",
        parameter: toolParam.name,
        description: "Parameter exists in tool but not in API",
      });
    }
  });

  return audit;
}

/**
 * Get all parameters from an API endpoint
 */
function getAllParameters(apiEndpoint) {
  const allParams = [];

  // Add path parameters
  if (apiEndpoint.parameters?.path) {
    allParams.push(...apiEndpoint.parameters.path);
  }

  // Add query parameters
  if (apiEndpoint.parameters?.query) {
    allParams.push(...apiEndpoint.parameters.query);
  }

  // Add body parameters
  if (apiEndpoint.parameters?.body) {
    allParams.push(...apiEndpoint.parameters.body);
  }

  return allParams;
}

/**
 * Get total parameter count for an endpoint
 */
function getTotalParameterCount(apiEndpoint) {
  return getAllParameters(apiEndpoint).length;
}

/**
 * Generate comprehensive audit report
 */
function generateAuditReport(auditResults) {
  console.log("=== MCP TOOLS AUDIT REPORT ===");
  console.log("");

  console.log("📊 OVERVIEW:");
  console.log(`Total API endpoints: ${auditResults.totalApiEndpoints}`);
  console.log(`Total existing tools: ${auditResults.totalToolsFound}`);
  console.log(`Tools with issues: ${auditResults.toolsWithIssues}`);
  console.log(`Missing tools: ${auditResults.missingTools.length}`);
  console.log("");

  if (auditResults.missingTools.length > 0) {
    console.log("❌ MISSING TOOLS:");
    auditResults.missingTools.slice(0, 10).forEach((missing) => {
      console.log(`  • ${missing.method} ${missing.path} - ${missing.summary}`);
    });
    if (auditResults.missingTools.length > 10) {
      console.log(`  ... and ${auditResults.missingTools.length - 10} more`);
    }
    console.log("");
  }

  if (auditResults.toolIssues.length > 0) {
    console.log("⚠️  TOOL ISSUES:");
    auditResults.toolIssues.slice(0, 5).forEach((issue) => {
      console.log(`  • ${issue.toolName}: ${issue.issues.length} issues`);
      issue.issues.slice(0, 3).forEach((detail) => {
        console.log(`    - ${detail.type}: ${detail.parameter || "N/A"}`);
      });
    });
    if (auditResults.toolIssues.length > 5) {
      console.log(
        `  ... and ${auditResults.toolIssues.length - 5} more tools with issues`
      );
    }
    console.log("");
  }

  // Calculate coverage percentage
  const coveragePercentage = (
    (auditResults.totalToolsFound / auditResults.totalApiEndpoints) *
    100
  ).toFixed(1);
  console.log(
    `📈 COVERAGE: ${coveragePercentage}% of API endpoints have corresponding tools`
  );
  console.log("");
}

// Run the audit
if (import.meta.url === `file://${process.argv[1]}`) {
  auditMCPTools();
}

export { auditMCPTools };