first commit

app/chrome-extension/inject-scripts/click-helper.js

@@ -0,0 +1,233 @@
+/* eslint-disable */
+// click-helper.js
+// This script is injected into the page to handle click operations
+
+if (window.__CLICK_HELPER_INITIALIZED__) {
+  // Already initialized, skip
+} else {
+  window.__CLICK_HELPER_INITIALIZED__ = true;
+  /**
+   * Click on an element matching the selector or at specific coordinates
+   * @param {string} selector - CSS selector for the element to click
+   * @param {boolean} waitForNavigation - Whether to wait for navigation to complete after click
+   * @param {number} timeout - Timeout in milliseconds for waiting for the element or navigation
+   * @param {Object} coordinates - Optional coordinates for clicking at a specific position
+   * @param {number} coordinates.x - X coordinate relative to the viewport
+   * @param {number} coordinates.y - Y coordinate relative to the viewport
+   * @returns {Promise<Object>} - Result of the click operation
+   */
+  async function clickElement(
+    selector,
+    waitForNavigation = false,
+    timeout = 5000,
+    coordinates = null,
+  ) {
+    try {
+      let element = null;
+      let elementInfo = null;
+      let clickX, clickY;
+
+      if (coordinates && typeof coordinates.x === 'number' && typeof coordinates.y === 'number') {
+        clickX = coordinates.x;
+        clickY = coordinates.y;
+
+        element = document.elementFromPoint(clickX, clickY);
+
+        if (element) {
+          const rect = element.getBoundingClientRect();
+          elementInfo = {
+            tagName: element.tagName,
+            id: element.id,
+            className: element.className,
+            text: element.textContent?.trim().substring(0, 100) || '',
+            href: element.href || null,
+            type: element.type || null,
+            isVisible: true,
+            rect: {
+              x: rect.x,
+              y: rect.y,
+              width: rect.width,
+              height: rect.height,
+              top: rect.top,
+              right: rect.right,
+              bottom: rect.bottom,
+              left: rect.left,
+            },
+            clickMethod: 'coordinates',
+            clickPosition: { x: clickX, y: clickY },
+          };
+        } else {
+          elementInfo = {
+            clickMethod: 'coordinates',
+            clickPosition: { x: clickX, y: clickY },
+            warning: 'No element found at the specified coordinates',
+          };
+        }
+      } else {
+        element = document.querySelector(selector);
+        if (!element) {
+          return {
+            error: `Element with selector "${selector}" not found`,
+          };
+        }
+
+        const rect = element.getBoundingClientRect();
+        elementInfo = {
+          tagName: element.tagName,
+          id: element.id,
+          className: element.className,
+          text: element.textContent?.trim().substring(0, 100) || '',
+          href: element.href || null,
+          type: element.type || null,
+          isVisible: true,
+          rect: {
+            x: rect.x,
+            y: rect.y,
+            width: rect.width,
+            height: rect.height,
+            top: rect.top,
+            right: rect.right,
+            bottom: rect.bottom,
+            left: rect.left,
+          },
+          clickMethod: 'selector',
+        };
+
+        // First sroll so that the element is in view, then check visibility.
+        element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' });
+        await new Promise((resolve) => setTimeout(resolve, 100));
+        elementInfo.isVisible = isElementVisible(element);
+        if (!elementInfo.isVisible) {
+          return {
+            error: `Element with selector "${selector}" is not visible`,
+            elementInfo,
+          };
+        }
+
+        const updatedRect = element.getBoundingClientRect();
+        clickX = updatedRect.left + updatedRect.width / 2;
+        clickY = updatedRect.top + updatedRect.height / 2;
+      }
+
+      let navigationPromise;
+      if (waitForNavigation) {
+        navigationPromise = new Promise((resolve) => {
+          const beforeUnloadListener = () => {
+            window.removeEventListener('beforeunload', beforeUnloadListener);
+            resolve(true);
+          };
+          window.addEventListener('beforeunload', beforeUnloadListener);
+
+          setTimeout(() => {
+            window.removeEventListener('beforeunload', beforeUnloadListener);
+            resolve(false);
+          }, timeout);
+        });
+      }
+
+      if (element && elementInfo.clickMethod === 'selector') {
+        element.click();
+      } else {
+        simulateClick(clickX, clickY);
+      }
+
+      // Wait for navigation if needed
+      let navigationOccurred = false;
+      if (waitForNavigation) {
+        navigationOccurred = await navigationPromise;
+      }
+
+      return {
+        success: true,
+        message: 'Element clicked successfully',
+        elementInfo,
+        navigationOccurred,
+      };
+    } catch (error) {
+      return {
+        error: `Error clicking element: ${error.message}`,
+      };
+    }
+  }
+
+  /**
+   * Simulate a mouse click at specific coordinates
+   * @param {number} x - X coordinate relative to the viewport
+   * @param {number} y - Y coordinate relative to the viewport
+   */
+  function simulateClick(x, y) {
+    const clickEvent = new MouseEvent('click', {
+      view: window,
+      bubbles: true,
+      cancelable: true,
+      clientX: x,
+      clientY: y,
+    });
+
+    const element = document.elementFromPoint(x, y);
+
+    if (element) {
+      element.dispatchEvent(clickEvent);
+    } else {
+      document.dispatchEvent(clickEvent);
+    }
+  }
+
+  /**
+   * Check if an element is visible
+   * @param {Element} element - The element to check
+   * @returns {boolean} - Whether the element is visible
+   */
+  function isElementVisible(element) {
+    if (!element) return false;
+
+    const style = window.getComputedStyle(element);
+    if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
+      return false;
+    }
+
+    const rect = element.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) {
+      return false;
+    }
+
+    if (
+      rect.bottom < 0 ||
+      rect.top > window.innerHeight ||
+      rect.right < 0 ||
+      rect.left > window.innerWidth
+    ) {
+      return false;
+    }
+
+    const centerX = rect.left + rect.width / 2;
+    const centerY = rect.top + rect.height / 2;
+
+    const elementAtPoint = document.elementFromPoint(centerX, centerY);
+    if (!elementAtPoint) return false;
+
+    return element === elementAtPoint || element.contains(elementAtPoint);
+  }
+
+  // Listen for messages from the extension
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    if (request.action === 'clickElement') {
+      clickElement(
+        request.selector,
+        request.waitForNavigation,
+        request.timeout,
+        request.coordinates,
+      )
+        .then(sendResponse)
+        .catch((error) => {
+          sendResponse({
+            error: `Unexpected error: ${error.message}`,
+          });
+        });
+      return true; // Indicates async response
+    } else if (request.action === 'chrome_click_element_ping') {
+      sendResponse({ status: 'pong' });
+      return false;
+    }
+  });
+}

app/chrome-extension/inject-scripts/fill-helper.js

@@ -0,0 +1,205 @@
+/* eslint-disable */
+// fill-helper.js
+// This script is injected into the page to handle form filling operations
+
+if (window.__FILL_HELPER_INITIALIZED__) {
+  // Already initialized, skip
+} else {
+  window.__FILL_HELPER_INITIALIZED__ = true;
+  /**
+   * Fill an input element with the specified value
+   * @param {string} selector - CSS selector for the element to fill
+   * @param {string} value - Value to fill into the element
+   * @returns {Promise<Object>} - Result of the fill operation
+   */
+  async function fillElement(selector, value) {
+    try {
+      // Find the element
+      const element = document.querySelector(selector);
+      if (!element) {
+        return {
+          error: `Element with selector "${selector}" not found`,
+        };
+      }
+
+      // Get element information
+      const rect = element.getBoundingClientRect();
+      const elementInfo = {
+        tagName: element.tagName,
+        id: element.id,
+        className: element.className,
+        type: element.type || null,
+        isVisible: isElementVisible(element),
+        rect: {
+          x: rect.x,
+          y: rect.y,
+          width: rect.width,
+          height: rect.height,
+          top: rect.top,
+          right: rect.right,
+          bottom: rect.bottom,
+          left: rect.left,
+        },
+      };
+
+      // Check if element is visible
+      if (!elementInfo.isVisible) {
+        return {
+          error: `Element with selector "${selector}" is not visible`,
+          elementInfo,
+        };
+      }
+
+      // Check if element is an input, textarea, or select
+      const validTags = ['INPUT', 'TEXTAREA', 'SELECT'];
+      const validInputTypes = [
+        'text',
+        'email',
+        'password',
+        'number',
+        'search',
+        'tel',
+        'url',
+        'date',
+        'datetime-local',
+        'month',
+        'time',
+        'week',
+        'color',
+      ];
+
+      if (!validTags.includes(element.tagName)) {
+        return {
+          error: `Element with selector "${selector}" is not a fillable element (must be INPUT, TEXTAREA, or SELECT)`,
+          elementInfo,
+        };
+      }
+
+      // For input elements, check if the type is valid
+      if (
+        element.tagName === 'INPUT' &&
+        !validInputTypes.includes(element.type) &&
+        element.type !== null
+      ) {
+        return {
+          error: `Input element with selector "${selector}" has type "${element.type}" which is not fillable`,
+          elementInfo,
+        };
+      }
+
+      // Scroll element into view
+      element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' });
+      await new Promise((resolve) => setTimeout(resolve, 100));
+
+      // Focus the element
+      element.focus();
+
+      // Fill the element based on its type
+      if (element.tagName === 'SELECT') {
+        // For select elements, find the option with matching value or text
+        let optionFound = false;
+        for (const option of element.options) {
+          if (option.value === value || option.text === value) {
+            element.value = option.value;
+            optionFound = true;
+            break;
+          }
+        }
+
+        if (!optionFound) {
+          return {
+            error: `No option with value or text "${value}" found in select element`,
+            elementInfo,
+          };
+        }
+
+        // Trigger change event
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+      } else {
+        // For input and textarea elements
+
+        // Clear the current value
+        element.value = '';
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+
+        // Set the new value
+        element.value = value;
+
+        // Trigger input and change events
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+      }
+
+      // Blur the element
+      element.blur();
+
+      return {
+        success: true,
+        message: 'Element filled successfully',
+        elementInfo: {
+          ...elementInfo,
+          value: element.value, // Include the final value in the response
+        },
+      };
+    } catch (error) {
+      return {
+        error: `Error filling element: ${error.message}`,
+      };
+    }
+  }
+
+  /**
+   * Check if an element is visible
+   * @param {Element} element - The element to check
+   * @returns {boolean} - Whether the element is visible
+   */
+  function isElementVisible(element) {
+    if (!element) return false;
+
+    const style = window.getComputedStyle(element);
+    if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
+      return false;
+    }
+
+    const rect = element.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) {
+      return false;
+    }
+
+    // Check if element is within viewport
+    if (
+      rect.bottom < 0 ||
+      rect.top > window.innerHeight ||
+      rect.right < 0 ||
+      rect.left > window.innerWidth
+    ) {
+      return false;
+    }
+
+    // Check if element is actually visible at its center point
+    const centerX = rect.left + rect.width / 2;
+    const centerY = rect.top + rect.height / 2;
+
+    const elementAtPoint = document.elementFromPoint(centerX, centerY);
+    if (!elementAtPoint) return false;
+
+    return element === elementAtPoint || element.contains(elementAtPoint);
+  }
+
+  // Listen for messages from the extension
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    if (request.action === 'fillElement') {
+      fillElement(request.selector, request.value)
+        .then(sendResponse)
+        .catch((error) => {
+          sendResponse({
+            error: `Unexpected error: ${error.message}`,
+          });
+        });
+      return true; // Indicates async response
+    } else if (request.action === 'chrome_fill_or_select_ping') {
+      sendResponse({ status: 'pong' });
+      return false;
+    }
+  });
+}

app/chrome-extension/inject-scripts/inject-bridge.js

@@ -0,0 +1,65 @@
+/* eslint-disable */
+
+(() => {
+  // Prevent duplicate injection of the bridge itself.
+  if (window.__INJECT_SCRIPT_TOOL_UNIVERSAL_BRIDGE_LOADED__) return;
+  window.__INJECT_SCRIPT_TOOL_UNIVERSAL_BRIDGE_LOADED__ = true;
+  const EVENT_NAME = {
+    RESPONSE: 'chrome-mcp:response',
+    CLEANUP: 'chrome-mcp:cleanup',
+    EXECUTE: 'chrome-mcp:execute',
+  };
+  const pendingRequests = new Map();
+
+  const messageHandler = (request, _sender, sendResponse) => {
+    // --- Lifecycle Command ---
+    if (request.type === EVENT_NAME.CLEANUP) {
+      window.dispatchEvent(new CustomEvent(EVENT_NAME.CLEANUP));
+      // Acknowledge cleanup signal received, but don't hold the connection.
+      sendResponse({ success: true });
+      return true;
+    }
+
+    // --- Execution Command for MAIN world ---
+    if (request.targetWorld === 'MAIN') {
+      const requestId = `req-${Date.now()}-${Math.random()}`;
+      pendingRequests.set(requestId, sendResponse);
+
+      window.dispatchEvent(
+        new CustomEvent(EVENT_NAME.EXECUTE, {
+          detail: {
+            action: request.action,
+            payload: request.payload,
+            requestId: requestId,
+          },
+        }),
+      );
+      return true; // Async response is expected.
+    }
+    // Note: Requests for ISOLATED world are handled by the user's isolatedWorldCode script directly.
+    // This listener won't process them unless it's the only script in ISOLATED world.
+  };
+
+  chrome.runtime.onMessage.addListener(messageHandler);
+
+  // Listen for responses coming back from the MAIN world.
+  const responseHandler = (event) => {
+    const { requestId, data, error } = event.detail;
+    if (pendingRequests.has(requestId)) {
+      const sendResponse = pendingRequests.get(requestId);
+      sendResponse({ data, error });
+      pendingRequests.delete(requestId);
+    }
+  };
+  window.addEventListener(EVENT_NAME.RESPONSE, responseHandler);
+
+  // --- Self Cleanup ---
+  // When the cleanup signal arrives, this bridge must also clean itself up.
+  const cleanupHandler = () => {
+    chrome.runtime.onMessage.removeListener(messageHandler);
+    window.removeEventListener(EVENT_NAME.RESPONSE, responseHandler);
+    window.removeEventListener(EVENT_NAME.CLEANUP, cleanupHandler);
+    delete window.__INJECT_SCRIPT_TOOL_UNIVERSAL_BRIDGE_LOADED__;
+  };
+  window.addEventListener(EVENT_NAME.CLEANUP, cleanupHandler);
+})();

app/chrome-extension/inject-scripts/interactive-elements-helper.js

@@ -0,0 +1,354 @@
+/* eslint-disable */
+// interactive-elements-helper.js
+// This script is injected into the page to find interactive elements.
+// Final version by Calvin, featuring a multi-layered fallback strategy
+// and comprehensive element support, built on a performant and reliable core.
+
+(function () {
+  // Prevent re-initialization
+  if (window.__INTERACTIVE_ELEMENTS_HELPER_INITIALIZED__) {
+    return;
+  }
+  window.__INTERACTIVE_ELEMENTS_HELPER_INITIALIZED__ = true;
+
+  /**
+   * @typedef {Object} ElementInfo
+   * @property {string} type - The type of the element (e.g., 'button', 'link').
+   * @property {string} selector - A CSS selector to uniquely identify the element.
+   * @property {string} text - The visible text or accessible name of the element.
+   * @property {boolean} isInteractive - Whether the element is currently interactive.
+   * @property {Object} [coordinates] - The coordinates of the element if requested.
+   * @property {boolean} [disabled] - For elements that can be disabled.
+   * @property {string} [href] - For links.
+   * @property {boolean} [checked] - for checkboxes and radio buttons.
+   */
+
+  /**
+   * Configuration for element types and their corresponding selectors.
+   * Now more comprehensive with common ARIA roles.
+   */
+  const ELEMENT_CONFIG = {
+    button: 'button, input[type="button"], input[type="submit"], [role="button"]',
+    link: 'a[href], [role="link"]',
+    input:
+      'input:not([type="button"]):not([type="submit"]):not([type="checkbox"]):not([type="radio"])',
+    checkbox: 'input[type="checkbox"], [role="checkbox"]',
+    radio: 'input[type="radio"], [role="radio"]',
+    textarea: 'textarea',
+    select: 'select',
+    tab: '[role="tab"]',
+    // Generic interactive elements: combines tabindex, common roles, and explicit handlers.
+    // This is the key to finding custom-built interactive components.
+    interactive: `[onclick], [tabindex]:not([tabindex^="-"]), [role="menuitem"], [role="slider"], [role="option"], [role="treeitem"]`,
+  };
+
+  // A combined selector for ANY interactive element, used in the fallback logic.
+  const ANY_INTERACTIVE_SELECTOR = Object.values(ELEMENT_CONFIG).join(', ');
+
+  // --- Core Helper Functions ---
+
+  /**
+   * Checks if an element is genuinely visible on the page.
+   * "Visible" means it's not styled with display:none, visibility:hidden, etc.
+   * This check intentionally IGNORES whether the element is within the current viewport.
+   * @param {Element} el The element to check.
+   * @returns {boolean} True if the element is visible.
+   */
+  function isElementVisible(el) {
+    if (!el || !el.isConnected) return false;
+
+    const style = window.getComputedStyle(el);
+    if (
+      style.display === 'none' ||
+      style.visibility === 'hidden' ||
+      parseFloat(style.opacity) === 0
+    ) {
+      return false;
+    }
+
+    const rect = el.getBoundingClientRect();
+    return rect.width > 0 || rect.height > 0 || el.tagName === 'A'; // Allow zero-size anchors as they can still be navigated
+  }
+
+  /**
+   * Checks if an element is considered interactive (not disabled or hidden from accessibility).
+   * @param {Element} el The element to check.
+   * @returns {boolean} True if the element is interactive.
+   */
+  function isElementInteractive(el) {
+    if (el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true') {
+      return false;
+    }
+    if (el.closest('[aria-hidden="true"]')) {
+      return false;
+    }
+    return true;
+  }
+
+  /**
+   * Generates a reasonably stable CSS selector for a given element.
+   * @param {Element} el The element.
+   * @returns {string} A CSS selector.
+   */
+  function generateSelector(el) {
+    if (!(el instanceof Element)) return '';
+
+    if (el.id) {
+      const idSelector = `#${CSS.escape(el.id)}`;
+      if (document.querySelectorAll(idSelector).length === 1) return idSelector;
+    }
+
+    for (const attr of ['data-testid', 'data-cy', 'name']) {
+      const attrValue = el.getAttribute(attr);
+      if (attrValue) {
+        const attrSelector = `[${attr}="${CSS.escape(attrValue)}"]`;
+        if (document.querySelectorAll(attrSelector).length === 1) return attrSelector;
+      }
+    }
+
+    let path = '';
+    let current = el;
+    while (current && current.nodeType === Node.ELEMENT_NODE && current.tagName !== 'BODY') {
+      let selector = current.tagName.toLowerCase();
+      const parent = current.parentElement;
+      if (parent) {
+        const siblings = Array.from(parent.children).filter(
+          (child) => child.tagName === current.tagName,
+        );
+        if (siblings.length > 1) {
+          const index = siblings.indexOf(current) + 1;
+          selector += `:nth-of-type(${index})`;
+        }
+      }
+      path = path ? `${selector} > ${path}` : selector;
+      current = parent;
+    }
+    return path ? `body > ${path}` : 'body';
+  }
+
+  /**
+   * Finds the accessible name for an element (label, aria-label, etc.).
+   * @param {Element} el The element.
+   * @returns {string} The accessible name.
+   */
+  function getAccessibleName(el) {
+    const labelledby = el.getAttribute('aria-labelledby');
+    if (labelledby) {
+      const labelElement = document.getElementById(labelledby);
+      if (labelElement) return labelElement.textContent?.trim() || '';
+    }
+    const ariaLabel = el.getAttribute('aria-label');
+    if (ariaLabel) return ariaLabel.trim();
+    if (el.id) {
+      const label = document.querySelector(`label[for="${el.id}"]`);
+      if (label) return label.textContent?.trim() || '';
+    }
+    const parentLabel = el.closest('label');
+    if (parentLabel) return parentLabel.textContent?.trim() || '';
+    return (
+      el.getAttribute('placeholder') ||
+      el.getAttribute('value') ||
+      el.textContent?.trim() ||
+      el.getAttribute('title') ||
+      ''
+    );
+  }
+
+  /**
+   * Simple subsequence matching for fuzzy search.
+   * @param {string} text The text to search within.
+   * @param {string} query The query subsequence.
+   * @returns {boolean}
+   */
+  function fuzzyMatch(text, query) {
+    if (!text || !query) return false;
+    const lowerText = text.toLowerCase();
+    const lowerQuery = query.toLowerCase();
+    let textIndex = 0;
+    let queryIndex = 0;
+    while (textIndex < lowerText.length && queryIndex < lowerQuery.length) {
+      if (lowerText[textIndex] === lowerQuery[queryIndex]) {
+        queryIndex++;
+      }
+      textIndex++;
+    }
+    return queryIndex === lowerQuery.length;
+  }
+
+  /**
+   * Creates the standardized info object for an element.
+   * Modified to handle the new 'text' type from the final fallback.
+   */
+  function createElementInfo(el, type, includeCoordinates, isInteractiveOverride = null) {
+    const isActuallyInteractive = isElementInteractive(el);
+    const info = {
+      type,
+      selector: generateSelector(el),
+      text: getAccessibleName(el) || el.textContent?.trim(),
+      isInteractive: isInteractiveOverride !== null ? isInteractiveOverride : isActuallyInteractive,
+      disabled: el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true',
+    };
+    if (includeCoordinates) {
+      const rect = el.getBoundingClientRect();
+      info.coordinates = {
+        x: rect.left + rect.width / 2,
+        y: rect.top + rect.height / 2,
+        rect: {
+          x: rect.x,
+          y: rect.y,
+          width: rect.width,
+          height: rect.height,
+          top: rect.top,
+          right: rect.right,
+          bottom: rect.bottom,
+          left: rect.left,
+        },
+      };
+    }
+    return info;
+  }
+
+  /**
+   * [CORE UTILITY] Finds interactive elements based on a set of types.
+   * This is our high-performance Layer 1 search function.
+   */
+  function findInteractiveElements(options = {}) {
+    const { textQuery, includeCoordinates = true, types = Object.keys(ELEMENT_CONFIG) } = options;
+
+    const selectorsToFind = types
+      .map((type) => ELEMENT_CONFIG[type])
+      .filter(Boolean)
+      .join(', ');
+    if (!selectorsToFind) return [];
+
+    const targetElements = Array.from(document.querySelectorAll(selectorsToFind));
+    const uniqueElements = new Set(targetElements);
+    const results = [];
+
+    for (const el of uniqueElements) {
+      if (!isElementVisible(el) || !isElementInteractive(el)) continue;
+
+      const accessibleName = getAccessibleName(el);
+      if (textQuery && !fuzzyMatch(accessibleName, textQuery)) continue;
+
+      let elementType = 'unknown';
+      for (const [type, typeSelector] of Object.entries(ELEMENT_CONFIG)) {
+        if (el.matches(typeSelector)) {
+          elementType = type;
+          break;
+        }
+      }
+      results.push(createElementInfo(el, elementType, includeCoordinates));
+    }
+    return results;
+  }
+
+  /**
+   * [ORCHESTRATOR] The main entry point that implements the 3-layer fallback logic.
+   * @param {object} options - The main search options.
+   * @returns {ElementInfo[]}
+   */
+  function findElementsByTextWithFallback(options = {}) {
+    const { textQuery, includeCoordinates = true } = options;
+
+    if (!textQuery) {
+      return findInteractiveElements({ ...options, types: Object.keys(ELEMENT_CONFIG) });
+    }
+
+    // --- Layer 1: High-reliability search for interactive elements matching text ---
+    let results = findInteractiveElements({ ...options, types: Object.keys(ELEMENT_CONFIG) });
+    if (results.length > 0) {
+      return results;
+    }
+
+    // --- Layer 2: Find text, then find its interactive ancestor ---
+    const lowerCaseText = textQuery.toLowerCase();
+    const xPath = `//text()[contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '${lowerCaseText}')]`;
+    const textNodes = document.evaluate(
+      xPath,
+      document,
+      null,
+      XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
+      null,
+    );
+
+    const interactiveElements = new Set();
+    if (textNodes.snapshotLength > 0) {
+      for (let i = 0; i < textNodes.snapshotLength; i++) {
+        const parentElement = textNodes.snapshotItem(i).parentElement;
+        if (parentElement) {
+          const interactiveAncestor = parentElement.closest(ANY_INTERACTIVE_SELECTOR);
+          if (
+            interactiveAncestor &&
+            isElementVisible(interactiveAncestor) &&
+            isElementInteractive(interactiveAncestor)
+          ) {
+            interactiveElements.add(interactiveAncestor);
+          }
+        }
+      }
+
+      if (interactiveElements.size > 0) {
+        return Array.from(interactiveElements).map((el) => {
+          let elementType = 'interactive';
+          for (const [type, typeSelector] of Object.entries(ELEMENT_CONFIG)) {
+            if (el.matches(typeSelector)) {
+              elementType = type;
+              break;
+            }
+          }
+          return createElementInfo(el, elementType, includeCoordinates);
+        });
+      }
+    }
+
+    // --- Layer 3: Final fallback, return any element containing the text ---
+    const leafElements = new Set();
+    for (let i = 0; i < textNodes.snapshotLength; i++) {
+      const parentElement = textNodes.snapshotItem(i).parentElement;
+      if (parentElement && isElementVisible(parentElement)) {
+        leafElements.add(parentElement);
+      }
+    }
+
+    const finalElements = Array.from(leafElements).filter((el) => {
+      return ![...leafElements].some((otherEl) => el !== otherEl && el.contains(otherEl));
+    });
+
+    return finalElements.map((el) => createElementInfo(el, 'text', includeCoordinates, true));
+  }
+
+  // --- Chrome Message Listener ---
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    if (request.action === 'getInteractiveElements') {
+      try {
+        let elements;
+        if (request.selector) {
+          // If a selector is provided, bypass the text-based logic and use a direct query.
+          const foundEls = Array.from(document.querySelectorAll(request.selector));
+          elements = foundEls.map((el) =>
+            createElementInfo(
+              el,
+              'selected',
+              request.includeCoordinates !== false,
+              isElementInteractive(el),
+            ),
+          );
+        } else {
+          // Otherwise, use our powerful multi-layered text search
+          elements = findElementsByTextWithFallback(request);
+        }
+        sendResponse({ success: true, elements });
+      } catch (error) {
+        console.error('Error in getInteractiveElements:', error);
+        sendResponse({ success: false, error: error.message });
+      }
+      return true; // Async response
+    } else if (request.action === 'chrome_get_interactive_elements_ping') {
+      sendResponse({ status: 'pong' });
+      return false;
+    }
+  });
+
+  console.log('Interactive elements helper script loaded');
+})();

app/chrome-extension/inject-scripts/keyboard-helper.js

@@ -0,0 +1,291 @@
+/* eslint-disable */
+// keyboard-helper.js
+// This script is injected into the page to handle keyboard event simulation
+
+if (window.__KEYBOARD_HELPER_INITIALIZED__) {
+  // Already initialized, skip
+} else {
+  window.__KEYBOARD_HELPER_INITIALIZED__ = true;
+
+  // A map for special keys to their KeyboardEvent properties
+  // Key names should be lowercase for matching
+  const SPECIAL_KEY_MAP = {
+    enter: { key: 'Enter', code: 'Enter', keyCode: 13 },
+    tab: { key: 'Tab', code: 'Tab', keyCode: 9 },
+    esc: { key: 'Escape', code: 'Escape', keyCode: 27 },
+    escape: { key: 'Escape', code: 'Escape', keyCode: 27 },
+    space: { key: ' ', code: 'Space', keyCode: 32 },
+    backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 },
+    delete: { key: 'Delete', code: 'Delete', keyCode: 46 },
+    del: { key: 'Delete', code: 'Delete', keyCode: 46 },
+    up: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
+    arrowup: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 },
+    down: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
+    arrowdown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 },
+    left: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
+    arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 },
+    right: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
+    arrowright: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 },
+    home: { key: 'Home', code: 'Home', keyCode: 36 },
+    end: { key: 'End', code: 'End', keyCode: 35 },
+    pageup: { key: 'PageUp', code: 'PageUp', keyCode: 33 },
+    pagedown: { key: 'PageDown', code: 'PageDown', keyCode: 34 },
+    insert: { key: 'Insert', code: 'Insert', keyCode: 45 },
+    // Function keys
+    ...Object.fromEntries(
+      Array.from({ length: 12 }, (_, i) => [
+        `f${i + 1}`,
+        { key: `F${i + 1}`, code: `F${i + 1}`, keyCode: 112 + i },
+      ]),
+    ),
+  };
+
+  const MODIFIER_KEYS = {
+    ctrl: 'ctrlKey',
+    control: 'ctrlKey',
+    alt: 'altKey',
+    shift: 'shiftKey',
+    meta: 'metaKey',
+    command: 'metaKey',
+    cmd: 'metaKey',
+  };
+
+  /**
+   * Parses a key string (e.g., "Ctrl+Shift+A", "Enter") into a main key and modifiers.
+   * @param {string} keyString - String representation of a single key press (can include modifiers).
+   * @returns { {key: string, code: string, keyCode: number, charCode?: number, modifiers: {ctrlKey:boolean, altKey:boolean, shiftKey:boolean, metaKey:boolean}} | null }
+   *          Returns null if the keyString is invalid or represents only modifiers.
+   */
+  function parseSingleKeyCombination(keyString) {
+    const parts = keyString.split('+').map((part) => part.trim().toLowerCase());
+    const modifiers = {
+      ctrlKey: false,
+      altKey: false,
+      shiftKey: false,
+      metaKey: false,
+    };
+    let mainKeyPart = null;
+
+    for (const part of parts) {
+      if (MODIFIER_KEYS[part]) {
+        modifiers[MODIFIER_KEYS[part]] = true;
+      } else if (mainKeyPart === null) {
+        // First non-modifier is the main key
+        mainKeyPart = part;
+      } else {
+        // Invalid format: multiple main keys in a single combination (e.g., "Ctrl+A+B")
+        console.error(`Invalid key combination string: ${keyString}. Multiple main keys found.`);
+        return null;
+      }
+    }
+
+    if (!mainKeyPart) {
+      // This case could happen if the keyString is something like "Ctrl+" or just "Ctrl"
+      // If the intent was to press JUST 'Control', the input should be 'Control' not 'Control+'
+      // Let's check if mainKeyPart is actually a modifier name used as a main key
+      if (Object.keys(MODIFIER_KEYS).includes(parts[parts.length - 1]) && parts.length === 1) {
+        mainKeyPart = parts[parts.length - 1]; // e.g. user wants to press "Control" key itself
+        // For "Control" key itself, key: "Control", code: "ControlLeft" (or Right)
+        if (mainKeyPart === 'ctrl' || mainKeyPart === 'control')
+          return { key: 'Control', code: 'ControlLeft', keyCode: 17, modifiers };
+        if (mainKeyPart === 'alt') return { key: 'Alt', code: 'AltLeft', keyCode: 18, modifiers };
+        if (mainKeyPart === 'shift')
+          return { key: 'Shift', code: 'ShiftLeft', keyCode: 16, modifiers };
+        if (mainKeyPart === 'meta' || mainKeyPart === 'command' || mainKeyPart === 'cmd')
+          return { key: 'Meta', code: 'MetaLeft', keyCode: 91, modifiers };
+      } else {
+        console.error(`Invalid key combination string: ${keyString}. No main key specified.`);
+        return null;
+      }
+    }
+
+    const specialKey = SPECIAL_KEY_MAP[mainKeyPart];
+    if (specialKey) {
+      return { ...specialKey, modifiers };
+    }
+
+    // For single characters or other unmapped keys
+    if (mainKeyPart.length === 1) {
+      const charCode = mainKeyPart.charCodeAt(0);
+      // If Shift is active and it's a letter, use the uppercase version for 'key'
+      // This mimics more closely how keyboards behave.
+      let keyChar = mainKeyPart;
+      if (modifiers.shiftKey && mainKeyPart.match(/^[a-z]$/i)) {
+        keyChar = mainKeyPart.toUpperCase();
+      }
+
+      return {
+        key: keyChar,
+        code: `Key${mainKeyPart.toUpperCase()}`, // 'a' -> KeyA, 'A' -> KeyA
+        keyCode: charCode,
+        charCode: charCode, // charCode is legacy, but some old systems might use it
+        modifiers,
+      };
+    }
+
+    console.error(`Unknown key: ${mainKeyPart} in string "${keyString}"`);
+    return null; // Or handle as an error
+  }
+
+  /**
+   * Simulates a single key press (keydown, (keypress), keyup) for a parsed key.
+   * @param { {key: string, code: string, keyCode: number, charCode?: number, modifiers: object} } parsedKeyInfo
+   * @param {Element} element - Target element.
+   * @returns {{success: boolean, error?: string}}
+   */
+  function dispatchKeyEvents(parsedKeyInfo, element) {
+    if (!parsedKeyInfo) return { success: false, error: 'Invalid key info provided for dispatch.' };
+
+    const { key, code, keyCode, charCode, modifiers } = parsedKeyInfo;
+
+    const eventOptions = {
+      key: key,
+      code: code,
+      bubbles: true,
+      cancelable: true,
+      composed: true, // Important for shadow DOM
+      view: window,
+      ...modifiers, // ctrlKey, altKey, shiftKey, metaKey
+      // keyCode/which are deprecated but often set for compatibility
+      keyCode: keyCode || (key.length === 1 ? key.charCodeAt(0) : 0),
+      which: keyCode || (key.length === 1 ? key.charCodeAt(0) : 0),
+    };
+
+    try {
+      const kdRes = element.dispatchEvent(new KeyboardEvent('keydown', eventOptions));
+
+      // keypress is deprecated, but simulate if it's a character key or Enter
+      // Only dispatch if keydown was not cancelled and it's a character producing key
+      if (kdRes && (key.length === 1 || key === 'Enter' || key === ' ')) {
+        const keypressOptions = { ...eventOptions };
+        if (charCode) keypressOptions.charCode = charCode;
+        element.dispatchEvent(new KeyboardEvent('keypress', keypressOptions));
+      }
+
+      element.dispatchEvent(new KeyboardEvent('keyup', eventOptions));
+      return { success: true };
+    } catch (error) {
+      console.error(`Error dispatching key events for "${key}":`, error);
+      return {
+        success: false,
+        error: `Error dispatching key events for "${key}": ${error.message}`,
+      };
+    }
+  }
+
+  /**
+   * Simulate keyboard events on an element or document
+   * @param {string} keysSequenceString - String representation of key(s) (e.g., "Enter", "Ctrl+C, A, B")
+   * @param {Element} targetElement - Element to dispatch events on (optional)
+   * @param {number} delay - Delay between key sequences in milliseconds (optional)
+   * @returns {Promise<Object>} - Result of the keyboard operation
+   */
+  async function simulateKeyboard(keysSequenceString, targetElement = null, delay = 0) {
+    try {
+      const element = targetElement || document.activeElement || document.body;
+
+      if (element !== document.activeElement && typeof element.focus === 'function') {
+        element.focus();
+        await new Promise((resolve) => setTimeout(resolve, 50)); // Small delay for focus
+      }
+
+      const keyCombinations = keysSequenceString
+        .split(',')
+        .map((k) => k.trim())
+        .filter((k) => k.length > 0);
+      const operationResults = [];
+
+      for (let i = 0; i < keyCombinations.length; i++) {
+        const comboString = keyCombinations[i];
+        const parsedKeyInfo = parseSingleKeyCombination(comboString);
+
+        if (!parsedKeyInfo) {
+          operationResults.push({
+            keyCombination: comboString,
+            success: false,
+            error: `Invalid key string or combination: ${comboString}`,
+          });
+          continue; // Skip to next combination in sequence
+        }
+
+        const dispatchResult = dispatchKeyEvents(parsedKeyInfo, element);
+        operationResults.push({
+          keyCombination: comboString,
+          ...dispatchResult,
+        });
+
+        if (dispatchResult.error) {
+          // Optionally, decide if sequence should stop on first error
+          // For now, we continue but log the error in results
+          console.warn(
+            `Failed to simulate key combination "${comboString}": ${dispatchResult.error}`,
+          );
+        }
+
+        if (delay > 0 && i < keyCombinations.length - 1) {
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+
+      // Check if all individual operations were successful
+      const overallSuccess = operationResults.every((r) => r.success);
+
+      return {
+        success: overallSuccess,
+        message: overallSuccess
+          ? `Keyboard events simulated successfully: ${keysSequenceString}`
+          : `Some keyboard events failed for: ${keysSequenceString}`,
+        results: operationResults, // Detailed results for each key combination
+        targetElement: {
+          tagName: element.tagName,
+          id: element.id,
+          className: element.className,
+          type: element.type, // if applicable e.g. for input
+        },
+      };
+    } catch (error) {
+      console.error('Error in simulateKeyboard:', error);
+      return {
+        success: false,
+        error: `Error simulating keyboard events: ${error.message}`,
+        results: [],
+      };
+    }
+  }
+
+  // Listener for messages from the extension
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    if (request.action === 'simulateKeyboard') {
+      let targetEl = null;
+      if (request.selector) {
+        targetEl = document.querySelector(request.selector);
+        if (!targetEl) {
+          sendResponse({
+            success: false,
+            error: `Element with selector "${request.selector}" not found`,
+            results: [],
+          });
+          return true; // Keep channel open for async response
+        }
+      }
+
+      simulateKeyboard(request.keys, targetEl, request.delay)
+        .then(sendResponse)
+        .catch((error) => {
+          // This catch is for unexpected errors in simulateKeyboard promise chain itself
+          console.error('Unexpected error in simulateKeyboard promise chain:', error);
+          sendResponse({
+            success: false,
+            error: `Unexpected error during keyboard simulation: ${error.message}`,
+            results: [],
+          });
+        });
+      return true; // Indicates async response is expected
+    } else if (request.action === 'chrome_keyboard_ping') {
+      sendResponse({ status: 'pong', initialized: true }); // Respond that it's initialized
+      return false; // Synchronous response
+    }
+    // Not our message, or no async response needed
+    return false;
+  });
+}

app/chrome-extension/inject-scripts/network-helper.js

@@ -0,0 +1,129 @@
+/* eslint-disable */
+/**
+ * Network Capture Helper
+ *
+ * This script helps replay network requests with the original cookies and headers.
+ */
+
+// Prevent duplicate initialization
+if (window.__NETWORK_CAPTURE_HELPER_INITIALIZED__) {
+  // Already initialized, skip
+} else {
+  window.__NETWORK_CAPTURE_HELPER_INITIALIZED__ = true;
+
+  /**
+   * Replay a network request
+   * @param {string} url - The URL to send the request to
+   * @param {string} method - The HTTP method to use
+   * @param {Object} headers - The headers to include in the request
+   * @param {any} body - The body of the request
+   * @param {number} timeout - Timeout in milliseconds (default: 30000)
+   * @returns {Promise<Object>} - The response data
+   */
+  async function replayNetworkRequest(url, method, headers, body, timeout = 30000) {
+    try {
+      // Create fetch options
+      const options = {
+        method: method,
+        headers: headers || {},
+        credentials: 'include', // Include cookies
+        mode: 'cors',
+        cache: 'no-cache',
+      };
+
+      // Add body for non-GET requests
+      if (method !== 'GET' && method !== 'HEAD' && body !== undefined) {
+        options.body = body;
+      }
+
+      // 创建一个带超时的 fetch
+      const fetchWithTimeout = async (url, options, timeout) => {
+        const controller = new AbortController();
+        const signal = controller.signal;
+
+        // 设置超时
+        const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+        try {
+          const response = await fetch(url, { ...options, signal });
+          clearTimeout(timeoutId);
+          return response;
+        } catch (error) {
+          clearTimeout(timeoutId);
+          throw error;
+        }
+      };
+
+      // 发送带超时的请求
+      const response = await fetchWithTimeout(url, options, timeout);
+
+      // Process response
+      const responseData = {
+        status: response.status,
+        statusText: response.statusText,
+        headers: {},
+      };
+
+      // Get response headers
+      response.headers.forEach((value, key) => {
+        responseData.headers[key] = value;
+      });
+
+      // Try to get response body based on content type
+      const contentType = response.headers.get('content-type') || '';
+
+      try {
+        if (contentType.includes('application/json')) {
+          responseData.body = await response.json();
+        } else if (
+          contentType.includes('text/') ||
+          contentType.includes('application/xml') ||
+          contentType.includes('application/javascript')
+        ) {
+          responseData.body = await response.text();
+        } else {
+          // For binary data, just indicate it was received but not parsed
+          responseData.body = '[Binary data not displayed]';
+        }
+      } catch (error) {
+        responseData.body = `[Error parsing response body: ${error.message}]`;
+      }
+
+      return {
+        success: true,
+        response: responseData,
+      };
+    } catch (error) {
+      console.error('Error replaying request:', error);
+      return {
+        success: false,
+        error: `Error replaying request: ${error.message}`,
+      };
+    }
+  }
+
+  // Listen for messages from the extension
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    // Respond to ping message
+    if (request.action === 'chrome_network_request_ping') {
+      sendResponse({ status: 'pong' });
+      return false; // Synchronous response
+    } else if (request.action === 'sendPureNetworkRequest') {
+      replayNetworkRequest(
+        request.url,
+        request.method,
+        request.headers,
+        request.body,
+        request.timeout,
+      )
+        .then(sendResponse)
+        .catch((error) => {
+          sendResponse({
+            success: false,
+            error: `Unexpected error: ${error.message}`,
+          });
+        });
+      return true; // Indicates async response
+    }
+  });
+}

app/chrome-extension/inject-scripts/screenshot-helper.js

@@ -0,0 +1,160 @@
+/* eslint-disable */
+/**
+ * Screenshot helper content script
+ * Handles page preparation, scrolling, element positioning, etc.
+ */
+
+if (window.__SCREENSHOT_HELPER_INITIALIZED__) {
+  // Already initialized, skip
+} else {
+  window.__SCREENSHOT_HELPER_INITIALIZED__ = true;
+
+  // Save original styles
+  let originalOverflowStyle = '';
+  let hiddenFixedElements = [];
+
+  /**
+   * Get fixed/sticky positioned elements
+   * @returns Array of fixed/sticky elements
+   */
+  function getFixedElements() {
+    const fixed = [];
+
+    document.querySelectorAll('*').forEach((el) => {
+      const htmlEl = el;
+      const style = window.getComputedStyle(htmlEl);
+      if (style.position === 'fixed' || style.position === 'sticky') {
+        // Filter out tiny or invisible elements, and elements that are part of the extension UI
+        if (
+          htmlEl.offsetWidth > 1 &&
+          htmlEl.offsetHeight > 1 &&
+          !htmlEl.id.startsWith('chrome-mcp-')
+        ) {
+          fixed.push({
+            element: htmlEl,
+            originalDisplay: htmlEl.style.display,
+            originalVisibility: htmlEl.style.visibility,
+          });
+        }
+      }
+    });
+    return fixed;
+  }
+
+  /**
+   * Hide fixed/sticky elements
+   */
+  function hideFixedElements() {
+    hiddenFixedElements = getFixedElements();
+    hiddenFixedElements.forEach((item) => {
+      item.element.style.display = 'none';
+    });
+  }
+
+  /**
+   * Restore fixed/sticky elements
+   */
+  function showFixedElements() {
+    hiddenFixedElements.forEach((item) => {
+      item.element.style.display = item.originalDisplay || '';
+    });
+    hiddenFixedElements = [];
+  }
+
+  // Listen for messages from the extension
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    // Respond to ping message
+    if (request.action === 'chrome_screenshot_ping') {
+      sendResponse({ status: 'pong' });
+      return false; // Synchronous response
+    }
+
+    // Prepare page for capture
+    else if (request.action === 'preparePageForCapture') {
+      originalOverflowStyle = document.documentElement.style.overflow;
+      document.documentElement.style.overflow = 'hidden'; // Hide main scrollbar
+      if (request.options?.fullPage) {
+        // Only hide fixed elements for full page to avoid flicker
+        hideFixedElements();
+      }
+      // Give styles a moment to apply
+      setTimeout(() => {
+        sendResponse({ success: true });
+      }, 50);
+      return true; // Async response
+    }
+
+    // Get page details
+    else if (request.action === 'getPageDetails') {
+      const body = document.body;
+      const html = document.documentElement;
+      sendResponse({
+        totalWidth: Math.max(
+          body.scrollWidth,
+          body.offsetWidth,
+          html.clientWidth,
+          html.scrollWidth,
+          html.offsetWidth,
+        ),
+        totalHeight: Math.max(
+          body.scrollHeight,
+          body.offsetHeight,
+          html.clientHeight,
+          html.scrollHeight,
+          html.offsetHeight,
+        ),
+        viewportWidth: window.innerWidth,
+        viewportHeight: window.innerHeight,
+        devicePixelRatio: window.devicePixelRatio || 1,
+        currentScrollX: window.scrollX,
+        currentScrollY: window.scrollY,
+      });
+    }
+
+    // Get element details
+    else if (request.action === 'getElementDetails') {
+      const element = document.querySelector(request.selector);
+      if (element) {
+        element.scrollIntoView({ behavior: 'instant', block: 'nearest', inline: 'nearest' });
+        setTimeout(() => {
+          // Wait for scroll
+          const rect = element.getBoundingClientRect();
+          sendResponse({
+            rect: { x: rect.left, y: rect.top, width: rect.width, height: rect.height },
+            devicePixelRatio: window.devicePixelRatio || 1,
+          });
+        }, 200); // Increased delay for scrollIntoView
+        return true; // Async response
+      } else {
+        sendResponse({ error: `Element with selector "${request.selector}" not found.` });
+      }
+      return true; // Async response
+    }
+
+    // Scroll page
+    else if (request.action === 'scrollPage') {
+      window.scrollTo({ left: request.x, top: request.y, behavior: 'instant' });
+      // Wait for scroll and potential reflows/lazy-loading
+      setTimeout(() => {
+        sendResponse({
+          success: true,
+          newScrollX: window.scrollX,
+          newScrollY: window.scrollY,
+        });
+      }, request.scrollDelay || 300); // Configurable delay
+      return true; // Async response
+    }
+
+    // Reset page
+    else if (request.action === 'resetPageAfterCapture') {
+      document.documentElement.style.overflow = originalOverflowStyle;
+      showFixedElements();
+      if (typeof request.scrollX !== 'undefined' && typeof request.scrollY !== 'undefined') {
+        window.scrollTo({ left: request.scrollX, top: request.scrollY, behavior: 'instant' });
+      }
+      sendResponse({ success: true });
+    }
+
+    return false; // Synchronous response
+  });
+}