mirror of
https://github.com/aaronjmars/opendia.git
synced 2025-12-29 16:16:00 +00:00
2483 lines
72 KiB
JavaScript
2483 lines
72 KiB
JavaScript
// Enhanced Browser Automation Content Script with Anti-Detection
|
|
console.log("OpenDia enhanced content script loaded");
|
|
|
|
// Enhanced Pattern Database with Twitter-First Priority
|
|
const ENHANCED_PATTERNS = {
|
|
// Authentication patterns
|
|
auth: {
|
|
login: {
|
|
input: [
|
|
"[type='email']",
|
|
"[name*='username' i]",
|
|
"[placeholder*='email' i]",
|
|
"[name*='login' i]",
|
|
],
|
|
password: ["[type='password']", "[name*='password' i]"],
|
|
submit: [
|
|
"[type='submit']",
|
|
"button[form]",
|
|
".login-btn",
|
|
"[aria-label*='login' i]",
|
|
],
|
|
confidence: 0.9,
|
|
},
|
|
signup: {
|
|
input: [
|
|
"[name*='register' i]",
|
|
"[placeholder*='signup' i]",
|
|
"[name*='email' i]",
|
|
],
|
|
submit: ["[href*='signup']", ".signup-btn", "[aria-label*='register' i]"],
|
|
confidence: 0.85,
|
|
},
|
|
},
|
|
|
|
// Content creation patterns - Twitter FIRST
|
|
content: {
|
|
post_create: {
|
|
textarea: [
|
|
"[data-testid='tweetTextarea_0']", // Twitter FIRST (most specific)
|
|
"[aria-label='Post text']", // Twitter specific
|
|
"[contenteditable='true']", // Generic last
|
|
"textarea[placeholder*='post' i]",
|
|
"[data-text='true']",
|
|
],
|
|
submit: [
|
|
"[data-testid='tweetButtonInline']", // Twitter inline
|
|
"[data-testid='tweetButton']", // Twitter main
|
|
".post-btn",
|
|
".publish-btn",
|
|
"[aria-label*='post' i]",
|
|
],
|
|
confidence: 0.95,
|
|
},
|
|
comment: {
|
|
textarea: [
|
|
"textarea[placeholder*='comment' i]",
|
|
"[role='textbox']",
|
|
"[placeholder*='reply' i]",
|
|
],
|
|
submit: [
|
|
".comment-btn",
|
|
"[aria-label*='comment' i]",
|
|
"[aria-label*='reply' i]",
|
|
],
|
|
confidence: 0.8,
|
|
},
|
|
},
|
|
|
|
// Search patterns
|
|
search: {
|
|
global: {
|
|
input: [
|
|
"[data-testid='SearchBox_Search_Input']", // Twitter search first
|
|
"[type='search']",
|
|
"[role='searchbox']",
|
|
"[placeholder*='search' i]",
|
|
"[name*='search' i]",
|
|
],
|
|
submit: [
|
|
"[aria-label*='search' i]",
|
|
".search-btn",
|
|
"button[type='submit']",
|
|
],
|
|
confidence: 0.85,
|
|
},
|
|
},
|
|
|
|
// Navigation patterns
|
|
nav: {
|
|
menu: {
|
|
toggle: [
|
|
"[aria-label*='menu' i]",
|
|
".menu-btn",
|
|
".hamburger",
|
|
"[data-toggle='menu']",
|
|
],
|
|
items: ["nav a", ".nav-item", "[role='menuitem']"],
|
|
confidence: 0.8,
|
|
},
|
|
},
|
|
|
|
// Form patterns
|
|
form: {
|
|
submit: {
|
|
button: [
|
|
"[type='submit']",
|
|
"button[form]",
|
|
".submit-btn",
|
|
"[aria-label*='submit' i]",
|
|
],
|
|
confidence: 0.85,
|
|
},
|
|
reset: {
|
|
button: ["[type='reset']", ".reset-btn", "[aria-label*='reset' i]"],
|
|
confidence: 0.8,
|
|
},
|
|
},
|
|
};
|
|
|
|
// Anti-Detection Platform Configuration
|
|
const ANTI_DETECTION_PLATFORMS = {
|
|
"twitter.com": {
|
|
selectors: {
|
|
textarea: "[data-testid='tweetTextarea_0']",
|
|
submit: "[data-testid='tweetButtonInline'], [data-testid='tweetButton']",
|
|
},
|
|
bypassMethod: "twitter_direct",
|
|
},
|
|
"x.com": {
|
|
selectors: {
|
|
textarea: "[data-testid='tweetTextarea_0']",
|
|
submit: "[data-testid='tweetButtonInline'], [data-testid='tweetButton']",
|
|
},
|
|
bypassMethod: "twitter_direct",
|
|
},
|
|
// Add other platforms that need special handling
|
|
"linkedin.com": {
|
|
selectors: {
|
|
textarea: "[contenteditable='true'][role='textbox']",
|
|
submit: "[data-control-name='share.post']",
|
|
},
|
|
bypassMethod: "linkedin_direct",
|
|
},
|
|
"facebook.com": {
|
|
selectors: {
|
|
textarea: "[contenteditable='true'][data-text='true']",
|
|
submit: "[data-testid='react-composer-post-button']",
|
|
},
|
|
bypassMethod: "facebook_direct",
|
|
},
|
|
};
|
|
|
|
// Legacy pattern database for backward compatibility
|
|
const PATTERN_DATABASE = {
|
|
twitter: {
|
|
domains: ["twitter.com", "x.com"],
|
|
patterns: {
|
|
post_tweet: {
|
|
textarea:
|
|
"[data-testid='tweetTextarea_0'], [contenteditable='true'][data-text='true']",
|
|
submit:
|
|
"[data-testid='tweetButtonInline'], [data-testid='tweetButton']",
|
|
confidence: 0.95,
|
|
},
|
|
search: {
|
|
input:
|
|
"[data-testid='SearchBox_Search_Input'], input[placeholder*='search' i]",
|
|
submit: "[data-testid='SearchBox_Search_Button']",
|
|
confidence: 0.9,
|
|
},
|
|
},
|
|
},
|
|
github: {
|
|
domains: ["github.com"],
|
|
patterns: {
|
|
search: {
|
|
input: "input[placeholder*='Search' i].form-control",
|
|
submit: "button[type='submit']",
|
|
confidence: 0.85,
|
|
},
|
|
},
|
|
},
|
|
universal: {
|
|
search: {
|
|
selectors: [
|
|
"input[type='search']",
|
|
"input[placeholder*='search' i]",
|
|
"[role='searchbox']",
|
|
"input[name*='search' i]",
|
|
],
|
|
confidence: 0.6,
|
|
},
|
|
submit: {
|
|
selectors: [
|
|
"button[type='submit']:not([disabled])",
|
|
"input[type='submit']:not([disabled])",
|
|
"[role='button'][aria-label*='submit' i]",
|
|
],
|
|
confidence: 0.65,
|
|
},
|
|
},
|
|
};
|
|
|
|
// Token usage tracking and performance optimization
|
|
class TokenOptimizer {
|
|
constructor() {
|
|
this.metrics = JSON.parse(localStorage.getItem("opendia_metrics") || "[]");
|
|
this.successRates = JSON.parse(
|
|
localStorage.getItem("opendia_success_rates") || "{}"
|
|
);
|
|
this.recommendedMaxResults = 5;
|
|
this.lastCleanup = Date.now();
|
|
}
|
|
|
|
trackUsage(
|
|
operation,
|
|
tokenCount,
|
|
success,
|
|
pageType = "unknown",
|
|
method = "unknown"
|
|
) {
|
|
const metric = {
|
|
operation,
|
|
tokens: tokenCount,
|
|
success,
|
|
pageType,
|
|
method,
|
|
timestamp: Date.now(),
|
|
};
|
|
|
|
this.metrics.push(metric);
|
|
|
|
// Keep only last 100 metrics to prevent storage bloat
|
|
if (this.metrics.length > 100) {
|
|
this.metrics = this.metrics.slice(-100);
|
|
}
|
|
|
|
// Auto-adjust limits based on success rates
|
|
this.adjustLimits();
|
|
|
|
// Periodic cleanup (once per hour)
|
|
if (Date.now() - this.lastCleanup > 3600000) {
|
|
this.cleanup();
|
|
}
|
|
|
|
// Persist to localStorage
|
|
localStorage.setItem("opendia_metrics", JSON.stringify(this.metrics));
|
|
}
|
|
|
|
adjustLimits() {
|
|
const recent = this.metrics.slice(-20);
|
|
if (recent.length < 10) return;
|
|
|
|
const avgTokens =
|
|
recent.reduce((sum, m) => sum + m.tokens, 0) / recent.length;
|
|
const successRate = recent.filter((m) => m.success).length / recent.length;
|
|
|
|
// If using too many tokens but success rate is high, reduce max results
|
|
if (avgTokens > 200 && successRate > 0.8) {
|
|
this.recommendedMaxResults = Math.max(3, this.recommendedMaxResults - 1);
|
|
}
|
|
// If success rate is low, increase max results to get better matches
|
|
else if (successRate < 0.6) {
|
|
this.recommendedMaxResults = Math.min(10, this.recommendedMaxResults + 1);
|
|
}
|
|
}
|
|
|
|
cleanup() {
|
|
// Remove metrics older than 7 days
|
|
const cutoff = Date.now() - 7 * 24 * 60 * 60 * 1000;
|
|
this.metrics = this.metrics.filter((m) => m.timestamp > cutoff);
|
|
|
|
// Clean up success rates for rarely used combinations
|
|
const recentPageTypes = new Set(this.metrics.map((m) => m.pageType));
|
|
Object.keys(this.successRates).forEach((key) => {
|
|
const [pageType] = key.split(":");
|
|
if (!recentPageTypes.has(pageType)) {
|
|
delete this.successRates[key];
|
|
}
|
|
});
|
|
|
|
localStorage.setItem("opendia_metrics", JSON.stringify(this.metrics));
|
|
localStorage.setItem(
|
|
"opendia_success_rates",
|
|
JSON.stringify(this.successRates)
|
|
);
|
|
this.lastCleanup = Date.now();
|
|
}
|
|
|
|
trackMethodSuccess(pageType, intent, method, success) {
|
|
const key = `${pageType}:${intent}:${method}`;
|
|
if (!this.successRates[key]) {
|
|
this.successRates[key] = { attempts: 0, successes: 0 };
|
|
}
|
|
|
|
this.successRates[key].attempts++;
|
|
if (success) {
|
|
this.successRates[key].successes++;
|
|
}
|
|
|
|
localStorage.setItem(
|
|
"opendia_success_rates",
|
|
JSON.stringify(this.successRates)
|
|
);
|
|
}
|
|
|
|
getBestMethod(pageType, intent) {
|
|
const methods = [
|
|
"anti_detection_bypass",
|
|
"enhanced_pattern_match",
|
|
"pattern_database",
|
|
"viewport_scan",
|
|
"semantic_analysis",
|
|
];
|
|
|
|
return methods.sort((a, b) => {
|
|
const aKey = `${pageType}:${intent}:${a}`;
|
|
const bKey = `${pageType}:${intent}:${b}`;
|
|
const aRate = this.getSuccessRate(aKey);
|
|
const bRate = this.getSuccessRate(bKey);
|
|
return bRate - aRate;
|
|
})[0];
|
|
}
|
|
|
|
getSuccessRate(key) {
|
|
const data = this.successRates[key];
|
|
if (!data || data.attempts === 0) return 0;
|
|
return data.successes / data.attempts;
|
|
}
|
|
|
|
getRecommendedMaxResults() {
|
|
return this.recommendedMaxResults;
|
|
}
|
|
|
|
getAnalytics() {
|
|
const recent = this.metrics.slice(-50);
|
|
if (recent.length === 0) return { message: "No metrics available" };
|
|
|
|
const avgTokens =
|
|
recent.reduce((sum, m) => sum + m.tokens, 0) / recent.length;
|
|
const successRate = recent.filter((m) => m.success).length / recent.length;
|
|
const operationBreakdown = {};
|
|
const methodBreakdown = {};
|
|
|
|
recent.forEach((m) => {
|
|
operationBreakdown[m.operation] =
|
|
(operationBreakdown[m.operation] || 0) + 1;
|
|
methodBreakdown[m.method] = (methodBreakdown[m.method] || 0) + 1;
|
|
});
|
|
|
|
return {
|
|
totalOperations: recent.length,
|
|
avgTokensPerOperation: Math.round(avgTokens),
|
|
successRate: Math.round(successRate * 100),
|
|
recommendedMaxResults: this.recommendedMaxResults,
|
|
operationBreakdown,
|
|
methodBreakdown,
|
|
tokenSavings: this.calculateTokenSavings(recent),
|
|
};
|
|
}
|
|
|
|
calculateTokenSavings(metrics) {
|
|
// Estimate token savings vs naive approach
|
|
const actualTokens = metrics.reduce((sum, m) => sum + m.tokens, 0);
|
|
const estimatedNaiveTokens = metrics.length * 300; // Estimate for non-optimized approach
|
|
const savings = estimatedNaiveTokens - actualTokens;
|
|
const percentage = Math.round((savings / estimatedNaiveTokens) * 100);
|
|
|
|
return {
|
|
actualTokens,
|
|
estimatedNaiveTokens,
|
|
tokensSaved: savings,
|
|
percentageSaved: percentage,
|
|
};
|
|
}
|
|
}
|
|
|
|
class BrowserAutomation {
|
|
constructor() {
|
|
this.elementRegistry = new Map();
|
|
this.quickRegistry = new Map(); // For phase 1 quick matches
|
|
this.idCounter = 0;
|
|
this.quickIdCounter = 0;
|
|
this.tokenOptimizer = new TokenOptimizer();
|
|
this.setupMessageListener();
|
|
this.setupViewportAnalyzer();
|
|
}
|
|
|
|
setupViewportAnalyzer() {
|
|
this.visibilityMap = new Map();
|
|
this.observer = new IntersectionObserver(
|
|
(entries) => {
|
|
entries.forEach((entry) => {
|
|
this.visibilityMap.set(entry.target, {
|
|
visible: entry.isIntersecting,
|
|
ratio: entry.intersectionRatio,
|
|
});
|
|
});
|
|
},
|
|
{ threshold: [0, 0.1, 0.5, 1.0] }
|
|
);
|
|
}
|
|
|
|
setupMessageListener() {
|
|
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
|
this.handleMessage(message)
|
|
.then(sendResponse)
|
|
.catch((error) => {
|
|
sendResponse({
|
|
success: false,
|
|
error: error.message,
|
|
stack: error.stack,
|
|
});
|
|
});
|
|
return true; // Keep message channel open for async response
|
|
});
|
|
}
|
|
|
|
async handleMessage(message) {
|
|
const { action, data } = message;
|
|
const startTime = performance.now();
|
|
|
|
try {
|
|
let result;
|
|
switch (action) {
|
|
case "analyze":
|
|
result = await this.analyzePage(data);
|
|
break;
|
|
case "extract_content":
|
|
result = await this.extractContent(data);
|
|
break;
|
|
case "element_click":
|
|
result = await this.clickElement(data);
|
|
break;
|
|
case "element_fill":
|
|
// 🎯 CRITICAL: Anti-Detection Bypass Implementation
|
|
result = await this.fillElementWithAntiDetection(data);
|
|
break;
|
|
case "wait_for":
|
|
result = await this.waitForCondition(data);
|
|
break;
|
|
case "getPageInfo":
|
|
result = {
|
|
title: document.title,
|
|
url: window.location.href,
|
|
content: document.body.innerText,
|
|
};
|
|
break;
|
|
case "get_analytics":
|
|
result = this.tokenOptimizer.getAnalytics();
|
|
break;
|
|
case "clear_analytics":
|
|
localStorage.removeItem("opendia_metrics");
|
|
localStorage.removeItem("opendia_success_rates");
|
|
this.tokenOptimizer = new TokenOptimizer();
|
|
result = { message: "Analytics cleared successfully" };
|
|
break;
|
|
case "get_element_state":
|
|
const element = this.getElementById(data.element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${data.element_id}`);
|
|
}
|
|
result = {
|
|
element_id: data.element_id,
|
|
element_name: this.getElementName(element),
|
|
state: this.getElementState(element),
|
|
current_value: this.getElementValue(element),
|
|
};
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown action: ${action}`);
|
|
}
|
|
|
|
const executionTime = performance.now() - startTime;
|
|
const dataSize = new Blob([JSON.stringify(result)]).size;
|
|
|
|
return {
|
|
success: true,
|
|
data: result,
|
|
execution_time: Math.round(executionTime),
|
|
data_size: dataSize,
|
|
timestamp: Date.now(),
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
error: error.message,
|
|
stack: error.stack,
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
};
|
|
}
|
|
}
|
|
|
|
// 🎯 ANTI-DETECTION BYPASS METHOD
|
|
async fillElementWithAntiDetection({
|
|
element_id,
|
|
value,
|
|
clear_first = true,
|
|
force_focus = true,
|
|
}) {
|
|
const element = this.getElementById(element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${element_id}`);
|
|
}
|
|
|
|
const hostname = window.location.hostname;
|
|
const platformConfig = this.detectAntiDetectionPlatform(hostname);
|
|
|
|
if (platformConfig && this.shouldUseBypass(element, platformConfig)) {
|
|
console.log(
|
|
`🎯 Using ${platformConfig.bypassMethod} bypass for ${hostname}`
|
|
);
|
|
return await this.executeDirectBypass(
|
|
element,
|
|
value,
|
|
platformConfig,
|
|
element_id
|
|
);
|
|
} else {
|
|
// Use normal fillElement for non-detection platforms
|
|
console.log("🔧 Using standard fill method");
|
|
return await this.fillElementStandard({
|
|
element_id,
|
|
value,
|
|
clear_first,
|
|
force_focus,
|
|
});
|
|
}
|
|
}
|
|
|
|
detectAntiDetectionPlatform(hostname) {
|
|
// Check exact matches first
|
|
if (ANTI_DETECTION_PLATFORMS[hostname]) {
|
|
return ANTI_DETECTION_PLATFORMS[hostname];
|
|
}
|
|
|
|
// Check subdomain matches
|
|
for (const [domain, config] of Object.entries(ANTI_DETECTION_PLATFORMS)) {
|
|
if (hostname.includes(domain) || hostname.endsWith(`.${domain}`)) {
|
|
return config;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
shouldUseBypass(element, platformConfig) {
|
|
// Check if element matches platform-specific selectors
|
|
try {
|
|
const isTextarea =
|
|
document.querySelector(platformConfig.selectors.textarea) === element;
|
|
if (isTextarea) return true;
|
|
|
|
// Also check if element matches any textarea selector pattern
|
|
const textareaSelectors = platformConfig.selectors.textarea.split(", ");
|
|
return textareaSelectors.some((selector) => {
|
|
try {
|
|
return element.matches(selector);
|
|
} catch {
|
|
return false;
|
|
}
|
|
});
|
|
} catch (error) {
|
|
console.warn("Bypass detection failed:", error);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async executeDirectBypass(element, value, platformConfig, element_id) {
|
|
try {
|
|
console.log(`🐦 Executing ${platformConfig.bypassMethod} bypass`);
|
|
|
|
switch (platformConfig.bypassMethod) {
|
|
case "twitter_direct":
|
|
return await this.twitterDirectBypass(element, value, element_id);
|
|
case "linkedin_direct":
|
|
return await this.linkedinDirectBypass(element, value, element_id);
|
|
case "facebook_direct":
|
|
return await this.facebookDirectBypass(element, value, element_id);
|
|
default:
|
|
// Generic direct bypass
|
|
return await this.genericDirectBypass(element, value, element_id);
|
|
}
|
|
} catch (error) {
|
|
console.error(
|
|
"Direct bypass failed, falling back to standard method:",
|
|
error
|
|
);
|
|
return await this.fillElementStandard({
|
|
element_id,
|
|
value,
|
|
clear_first: true,
|
|
force_focus: true,
|
|
});
|
|
}
|
|
}
|
|
|
|
async twitterDirectBypass(element, value, element_id) {
|
|
// THE WORKING FORMULA FOR TWITTER:
|
|
// 1. Focus 2. Click 3. execCommand
|
|
console.log("🐦 Twitter direct bypass - focus+click+execCommand");
|
|
|
|
// Ensure element is in view
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((r) => setTimeout(r, 200));
|
|
|
|
// The magic sequence that bypasses Twitter detection
|
|
element.focus();
|
|
element.click();
|
|
const execResult = document.execCommand("insertText", false, value);
|
|
|
|
// Wait for React state to update
|
|
await new Promise((r) => setTimeout(r, 500));
|
|
|
|
// Verify success
|
|
const currentText = element.textContent || element.value || "";
|
|
const success = currentText.includes(value);
|
|
|
|
return {
|
|
success: success,
|
|
element_id: element_id,
|
|
value: value,
|
|
actual_value: currentText,
|
|
method: "twitter_direct_bypass",
|
|
execCommand_result: execResult,
|
|
element_name: this.getElementName(element),
|
|
};
|
|
}
|
|
|
|
async linkedinDirectBypass(element, value, element_id) {
|
|
console.log("💼 LinkedIn direct bypass");
|
|
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((r) => setTimeout(r, 200));
|
|
|
|
// LinkedIn-specific sequence
|
|
element.focus();
|
|
element.click();
|
|
|
|
// Clear existing content first for LinkedIn
|
|
if (element.textContent) {
|
|
document.execCommand("selectAll");
|
|
document.execCommand("delete");
|
|
}
|
|
|
|
const execResult = document.execCommand("insertText", false, value);
|
|
|
|
// LinkedIn needs more time for state updates
|
|
await new Promise((r) => setTimeout(r, 800));
|
|
|
|
const currentText = element.textContent || element.value || "";
|
|
const success = currentText.includes(value);
|
|
|
|
return {
|
|
success: success,
|
|
element_id: element_id,
|
|
value: value,
|
|
actual_value: currentText,
|
|
method: "linkedin_direct_bypass",
|
|
execCommand_result: execResult,
|
|
element_name: this.getElementName(element),
|
|
};
|
|
}
|
|
|
|
async facebookDirectBypass(element, value, element_id) {
|
|
console.log("📘 Facebook direct bypass");
|
|
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((r) => setTimeout(r, 200));
|
|
|
|
// Facebook-specific sequence
|
|
element.focus();
|
|
element.click();
|
|
|
|
// Facebook may need selection clearing
|
|
if (element.textContent) {
|
|
document.execCommand("selectAll");
|
|
document.execCommand("delete");
|
|
}
|
|
|
|
const execResult = document.execCommand("insertText", false, value);
|
|
|
|
// Trigger Facebook-specific events
|
|
element.dispatchEvent(new Event("input", { bubbles: true }));
|
|
element.dispatchEvent(new Event("change", { bubbles: true }));
|
|
|
|
await new Promise((r) => setTimeout(r, 600));
|
|
|
|
const currentText = element.textContent || element.value || "";
|
|
const success = currentText.includes(value);
|
|
|
|
return {
|
|
success: success,
|
|
element_id: element_id,
|
|
value: value,
|
|
actual_value: currentText,
|
|
method: "facebook_direct_bypass",
|
|
execCommand_result: execResult,
|
|
element_name: this.getElementName(element),
|
|
};
|
|
}
|
|
|
|
async genericDirectBypass(element, value, element_id) {
|
|
console.log("🔧 Generic direct bypass");
|
|
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((r) => setTimeout(r, 200));
|
|
|
|
// Generic direct sequence
|
|
element.focus();
|
|
element.click();
|
|
const execResult = document.execCommand("insertText", false, value);
|
|
|
|
await new Promise((r) => setTimeout(r, 500));
|
|
|
|
const currentText = element.textContent || element.value || "";
|
|
const success = currentText.includes(value);
|
|
|
|
return {
|
|
success: success,
|
|
element_id: element_id,
|
|
value: value,
|
|
actual_value: currentText,
|
|
method: "generic_direct_bypass",
|
|
execCommand_result: execResult,
|
|
element_name: this.getElementName(element),
|
|
};
|
|
}
|
|
|
|
// Standard fill method (unchanged for compatibility)
|
|
async fillElementStandard({
|
|
element_id,
|
|
value,
|
|
clear_first = true,
|
|
force_focus = true,
|
|
}) {
|
|
const element = this.getElementById(element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${element_id}`);
|
|
}
|
|
|
|
// Enhanced focus sequence for modern web apps
|
|
if (force_focus) {
|
|
await this.ensureProperFocus(element);
|
|
} else {
|
|
element.focus();
|
|
}
|
|
|
|
// Clear existing content if requested
|
|
if (clear_first) {
|
|
await this.clearElementContent(element);
|
|
}
|
|
|
|
// Fill the value with proper event sequence
|
|
await this.fillWithEvents(element, value);
|
|
|
|
// Validate the fill was successful
|
|
const actualValue = this.getElementValue(element);
|
|
const success = actualValue.includes(value);
|
|
|
|
return {
|
|
success,
|
|
element_id,
|
|
value,
|
|
actual_value: actualValue,
|
|
element_name: this.getElementName(element),
|
|
method: "standard_fill",
|
|
focus_applied: force_focus,
|
|
};
|
|
}
|
|
|
|
async analyzePage({
|
|
intent_hint,
|
|
phase = "discover",
|
|
focus_areas,
|
|
element_ids,
|
|
max_results = 5,
|
|
}) {
|
|
const startTime = performance.now();
|
|
|
|
// Two-phase approach
|
|
if (phase === "discover") {
|
|
return await this.quickDiscovery({ intent_hint, max_results });
|
|
} else if (phase === "detailed") {
|
|
return await this.detailedAnalysis({
|
|
intent_hint,
|
|
focus_areas,
|
|
element_ids,
|
|
max_results,
|
|
});
|
|
}
|
|
|
|
// Legacy single-phase approach for backward compatibility
|
|
return await this.legacyAnalysis({
|
|
intent_hint,
|
|
focus_area: focus_areas?.[0],
|
|
max_results,
|
|
});
|
|
}
|
|
|
|
async quickDiscovery({ intent_hint, max_results = 5 }) {
|
|
const startTime = performance.now();
|
|
|
|
// Detect page type and get basic metrics
|
|
const pageType = this.detectPageType();
|
|
const viewportElements = this.countViewportElements();
|
|
|
|
// Use token optimizer recommendations
|
|
const recommendedMaxResults =
|
|
this.tokenOptimizer.getRecommendedMaxResults();
|
|
max_results = Math.min(max_results, recommendedMaxResults);
|
|
|
|
// Get best method based on historical performance
|
|
const bestMethod = this.tokenOptimizer.getBestMethod(pageType, intent_hint);
|
|
|
|
// Try to find obvious matches using enhanced patterns
|
|
let quickMatches = [];
|
|
let usedMethod = "quick_discovery";
|
|
|
|
try {
|
|
// Check for anti-detection bypass first
|
|
const hostname = window.location.hostname;
|
|
const platformConfig = this.detectAntiDetectionPlatform(hostname);
|
|
|
|
if (
|
|
platformConfig &&
|
|
(intent_hint.includes("post") || intent_hint.includes("tweet"))
|
|
) {
|
|
const bypassResult = await this.tryAntiDetectionPatterns(
|
|
intent_hint,
|
|
platformConfig
|
|
);
|
|
if (bypassResult.confidence > 0.9) {
|
|
quickMatches = bypassResult.elements
|
|
.slice(0, 3)
|
|
.map((el) => this.compressElement(el, true));
|
|
usedMethod = "anti_detection_bypass";
|
|
}
|
|
}
|
|
|
|
// Fallback to enhanced patterns
|
|
if (
|
|
quickMatches.length === 0 &&
|
|
(bestMethod === "enhanced_pattern_match" ||
|
|
bestMethod === "pattern_database")
|
|
) {
|
|
const patternResult = await this.tryEnhancedPatterns(intent_hint);
|
|
if (patternResult.confidence > 0.7) {
|
|
quickMatches = patternResult.elements
|
|
.slice(0, 3)
|
|
.map((el) => this.compressElement(el, true));
|
|
usedMethod = "enhanced_patterns";
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn("Enhanced patterns failed:", error);
|
|
}
|
|
|
|
// If no pattern matches, do a quick viewport scan
|
|
if (quickMatches.length === 0) {
|
|
quickMatches = await this.quickViewportScan(intent_hint, 3);
|
|
usedMethod = "viewport_scan";
|
|
}
|
|
|
|
const intentMatch = this.scoreIntentMatch(intent_hint, quickMatches);
|
|
const suggestedAreas = this.suggestPhase2Areas(quickMatches, intent_hint);
|
|
const executionTime = Math.round(performance.now() - startTime);
|
|
|
|
const result = {
|
|
summary: {
|
|
page_type: pageType,
|
|
intent_match: intentMatch,
|
|
element_count: viewportElements,
|
|
viewport_elements: quickMatches.length,
|
|
suggested_phase2: suggestedAreas,
|
|
anti_detection_platform: this.detectAntiDetectionPlatform(
|
|
window.location.hostname
|
|
)
|
|
? window.location.hostname
|
|
: null,
|
|
},
|
|
quick_matches: quickMatches,
|
|
token_estimate: this.estimatePhase2Tokens(quickMatches),
|
|
method: usedMethod,
|
|
execution_time: executionTime,
|
|
intent_hint: intent_hint, // Add this for server.js compatibility
|
|
elements: quickMatches, // Add this for backward compatibility
|
|
};
|
|
|
|
// Track token usage and success
|
|
const tokenCount = this.estimateTokenUsage(result);
|
|
const success = quickMatches.length > 0 && intentMatch !== "none";
|
|
this.tokenOptimizer.trackUsage(
|
|
"page_analyze_discover",
|
|
tokenCount,
|
|
success,
|
|
pageType,
|
|
usedMethod
|
|
);
|
|
this.tokenOptimizer.trackMethodSuccess(
|
|
pageType,
|
|
intent_hint,
|
|
usedMethod,
|
|
success
|
|
);
|
|
|
|
return result;
|
|
}
|
|
|
|
async tryAntiDetectionPatterns(intent_hint, platformConfig) {
|
|
const elements = [];
|
|
|
|
// Try to find platform-specific elements
|
|
try {
|
|
const textareaElement = document.querySelector(
|
|
platformConfig.selectors.textarea
|
|
);
|
|
if (textareaElement && this.isLikelyVisible(textareaElement)) {
|
|
const elementId = this.registerElement(textareaElement);
|
|
elements.push({
|
|
id: elementId,
|
|
type: "textarea",
|
|
selector: platformConfig.selectors.textarea,
|
|
name: this.getElementName(textareaElement),
|
|
confidence: 0.95, // High confidence for anti-detection platforms
|
|
element: textareaElement,
|
|
});
|
|
}
|
|
|
|
const submitElement = document.querySelector(
|
|
platformConfig.selectors.submit
|
|
);
|
|
if (submitElement && this.isLikelyVisible(submitElement)) {
|
|
const elementId = this.registerElement(submitElement);
|
|
elements.push({
|
|
id: elementId,
|
|
type: "button",
|
|
selector: platformConfig.selectors.submit,
|
|
name: this.getElementName(submitElement),
|
|
confidence: 0.95,
|
|
element: submitElement,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
console.warn("Anti-detection pattern matching failed:", error);
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: elements.length > 0 ? 0.95 : 0,
|
|
method: "anti_detection_patterns",
|
|
platform: window.location.hostname,
|
|
};
|
|
}
|
|
|
|
async detailedAnalysis({
|
|
intent_hint,
|
|
focus_areas,
|
|
element_ids,
|
|
max_results = 10,
|
|
}) {
|
|
const startTime = performance.now();
|
|
const pageType = this.detectPageType();
|
|
|
|
// Use token optimizer recommendations
|
|
const recommendedMaxResults =
|
|
this.tokenOptimizer.getRecommendedMaxResults();
|
|
max_results = Math.min(max_results, recommendedMaxResults + 2); // Allow slightly more for detailed analysis
|
|
|
|
let elements = [];
|
|
let method = "detailed_analysis";
|
|
|
|
// Expand specific quick matches if provided
|
|
if (element_ids?.length) {
|
|
elements = await this.expandQuickMatches(element_ids);
|
|
method = "expanded_matches";
|
|
}
|
|
|
|
// Analyze specific focus areas
|
|
if (focus_areas?.length) {
|
|
const areaElements = await this.analyzeFocusAreas(
|
|
focus_areas,
|
|
intent_hint
|
|
);
|
|
elements = [...elements, ...areaElements];
|
|
method = elements.length > 0 ? "focus_area_analysis" : method;
|
|
}
|
|
|
|
// If no specific analysis requested, do full enhanced analysis
|
|
if (elements.length === 0) {
|
|
elements = await this.fullEnhancedAnalysis(intent_hint, max_results);
|
|
method = "full_enhanced_analysis";
|
|
}
|
|
|
|
// Deduplicate and enhance with metadata
|
|
elements = this.deduplicateElements(elements);
|
|
elements = await this.enhanceElementMetadata(elements);
|
|
|
|
// Apply compact fingerprinting
|
|
elements = elements
|
|
.slice(0, max_results)
|
|
.map((el) => this.compressElement(el, false));
|
|
|
|
const executionTime = Math.round(performance.now() - startTime);
|
|
const result = {
|
|
elements,
|
|
interaction_ready: elements.every((el) => el.conf > 50),
|
|
method,
|
|
execution_time: executionTime,
|
|
intent_hint: intent_hint, // Add this for server.js compatibility
|
|
};
|
|
|
|
// Track token usage and success
|
|
const tokenCount = this.estimateTokenUsage(result);
|
|
const success = elements.length > 0 && elements.some((el) => el.conf > 70);
|
|
this.tokenOptimizer.trackUsage(
|
|
"page_analyze_detailed",
|
|
tokenCount,
|
|
success,
|
|
pageType,
|
|
method
|
|
);
|
|
this.tokenOptimizer.trackMethodSuccess(
|
|
pageType,
|
|
intent_hint,
|
|
method,
|
|
success
|
|
);
|
|
|
|
return result;
|
|
}
|
|
|
|
async legacyAnalysis({ intent_hint, focus_area, max_results = 5 }) {
|
|
const startTime = performance.now();
|
|
let result;
|
|
|
|
try {
|
|
// Try enhanced patterns first
|
|
result = await this.tryEnhancedPatterns(intent_hint);
|
|
if (result.confidence > 0.8) {
|
|
return this.formatAnalysisResult(
|
|
result,
|
|
"enhanced_patterns",
|
|
startTime
|
|
);
|
|
}
|
|
} catch (error) {
|
|
console.warn("Enhanced patterns failed, trying legacy patterns:", error);
|
|
try {
|
|
// Fallback to legacy pattern database
|
|
result = await this.tryPatternDatabase(intent_hint);
|
|
if (result.confidence > 0.8) {
|
|
return this.formatAnalysisResult(
|
|
result,
|
|
"pattern_database",
|
|
startTime
|
|
);
|
|
}
|
|
} catch (legacyError) {
|
|
console.warn("Legacy pattern database failed:", legacyError);
|
|
}
|
|
}
|
|
|
|
// Final fallback to semantic analysis
|
|
result = await this.trySemanticAnalysis(intent_hint, focus_area);
|
|
return this.formatAnalysisResult(result, "semantic_analysis", startTime);
|
|
}
|
|
|
|
async tryEnhancedPatterns(intent_hint) {
|
|
const [category, action] = this.parseIntent(intent_hint);
|
|
const pattern = ENHANCED_PATTERNS[category]?.[action];
|
|
|
|
if (!pattern) {
|
|
return this.tryUniversalPatterns(intent_hint);
|
|
}
|
|
|
|
const elements = this.findPatternElements(pattern);
|
|
return {
|
|
elements: elements.slice(0, 3),
|
|
confidence: pattern.confidence,
|
|
method: "enhanced_pattern_match",
|
|
category,
|
|
action,
|
|
};
|
|
}
|
|
|
|
parseIntent(intent) {
|
|
const intentLower = intent.toLowerCase();
|
|
|
|
// Check for authentication patterns
|
|
if (
|
|
intentLower.includes("login") ||
|
|
intentLower.includes("sign in") ||
|
|
intentLower.includes("log in")
|
|
) {
|
|
return ["auth", "login"];
|
|
}
|
|
if (
|
|
intentLower.includes("signup") ||
|
|
intentLower.includes("sign up") ||
|
|
intentLower.includes("register") ||
|
|
intentLower.includes("create account")
|
|
) {
|
|
return ["auth", "signup"];
|
|
}
|
|
|
|
// Check for content creation patterns
|
|
if (
|
|
intentLower.includes("tweet") ||
|
|
intentLower.includes("post") ||
|
|
intentLower.includes("compose") ||
|
|
intentLower.includes("create") ||
|
|
intentLower.includes("write") ||
|
|
intentLower.includes("publish")
|
|
) {
|
|
return ["content", "post_create"];
|
|
}
|
|
if (intentLower.includes("comment") || intentLower.includes("reply")) {
|
|
return ["content", "comment"];
|
|
}
|
|
|
|
// Check for search patterns
|
|
if (
|
|
intentLower.includes("search") ||
|
|
intentLower.includes("find") ||
|
|
intentLower.includes("look for")
|
|
) {
|
|
return ["search", "global"];
|
|
}
|
|
|
|
// Check for navigation patterns
|
|
if (
|
|
intentLower.includes("menu") ||
|
|
intentLower.includes("navigation") ||
|
|
intentLower.includes("nav")
|
|
) {
|
|
return ["nav", "menu"];
|
|
}
|
|
|
|
// Check for form patterns
|
|
if (
|
|
intentLower.includes("submit") ||
|
|
intentLower.includes("send") ||
|
|
intentLower.includes("save")
|
|
) {
|
|
return ["form", "submit"];
|
|
}
|
|
if (
|
|
intentLower.includes("reset") ||
|
|
intentLower.includes("clear") ||
|
|
intentLower.includes("cancel")
|
|
) {
|
|
return ["form", "reset"];
|
|
}
|
|
|
|
// Fallback - try to infer from context
|
|
if (intentLower.includes("button") || intentLower.includes("click")) {
|
|
return ["form", "submit"];
|
|
}
|
|
if (
|
|
intentLower.includes("input") ||
|
|
intentLower.includes("field") ||
|
|
intentLower.includes("text")
|
|
) {
|
|
return ["content", "post_create"];
|
|
}
|
|
|
|
// Default fallback
|
|
return ["content", "post_create"]; // More useful default than search
|
|
}
|
|
|
|
findPatternElements(pattern) {
|
|
const elements = [];
|
|
|
|
for (const [elementType, selectors] of Object.entries(pattern)) {
|
|
if (elementType === "confidence") continue;
|
|
|
|
for (const selector of selectors) {
|
|
const element = document.querySelector(selector);
|
|
if (element && this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: elementType,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence || 0.8,
|
|
element: element,
|
|
});
|
|
break; // Take first match per element type
|
|
}
|
|
}
|
|
}
|
|
|
|
return elements;
|
|
}
|
|
|
|
tryUniversalPatterns(intent_hint) {
|
|
const intentLower = intent_hint.toLowerCase();
|
|
let selectors = [];
|
|
|
|
// Content creation patterns
|
|
if (
|
|
intentLower.includes("tweet") ||
|
|
intentLower.includes("post") ||
|
|
intentLower.includes("compose") ||
|
|
intentLower.includes("create") ||
|
|
intentLower.includes("write")
|
|
) {
|
|
selectors = [
|
|
"[data-testid='tweetTextarea_0']", // Twitter first!
|
|
"[contenteditable='true']",
|
|
"textarea[placeholder*='tweet' i]",
|
|
"textarea[placeholder*='post' i]",
|
|
"textarea[placeholder*='what' i]",
|
|
"[data-text='true']",
|
|
"[role='textbox']",
|
|
"textarea:not([style*='display: none'])",
|
|
];
|
|
}
|
|
// Authentication patterns
|
|
else if (intentLower.includes("login") || intentLower.includes("sign in")) {
|
|
selectors = [
|
|
"[type='email']",
|
|
"[name*='username' i]",
|
|
"[placeholder*='email' i]",
|
|
"[placeholder*='username' i]",
|
|
"input[name*='login' i]",
|
|
];
|
|
} else if (
|
|
intentLower.includes("signup") ||
|
|
intentLower.includes("register")
|
|
) {
|
|
selectors = [
|
|
"[href*='signup']",
|
|
".signup-btn",
|
|
"[aria-label*='register' i]",
|
|
"button[data-testid*='signup' i]",
|
|
"a[href*='register']",
|
|
];
|
|
}
|
|
// Search patterns
|
|
else if (intentLower.includes("search") || intentLower.includes("find")) {
|
|
selectors = [
|
|
"[data-testid='SearchBox_Search_Input']", // Twitter search first
|
|
"[type='search']",
|
|
"[role='searchbox']",
|
|
"[placeholder*='search' i]",
|
|
"[data-testid*='search' i]",
|
|
"input[name*='search' i]",
|
|
];
|
|
}
|
|
// Generic fallback - look for interactive elements
|
|
else {
|
|
selectors = [
|
|
"button:not([disabled])",
|
|
"[contenteditable='true']",
|
|
"textarea",
|
|
"[type='submit']",
|
|
"[role='button']",
|
|
"input[type='text']",
|
|
];
|
|
}
|
|
|
|
const elements = [];
|
|
|
|
for (const selector of selectors) {
|
|
const foundElements = document.querySelectorAll(selector);
|
|
for (const element of foundElements) {
|
|
if (this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence:
|
|
0.5 + this.calculateConfidence(element, intent_hint) * 0.3,
|
|
element: element,
|
|
});
|
|
if (elements.length >= 3) break; // Limit to 3 elements
|
|
}
|
|
}
|
|
if (elements.length >= 3) break;
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence:
|
|
elements.length > 0
|
|
? Math.max(...elements.map((e) => e.confidence))
|
|
: 0,
|
|
method: "universal_pattern",
|
|
};
|
|
}
|
|
|
|
async tryPatternDatabase(intentHint) {
|
|
const hostname = window.location.hostname;
|
|
const siteKey = this.detectSite(hostname);
|
|
|
|
if (siteKey === "universal") {
|
|
return this.getUniversalPattern(intentHint);
|
|
}
|
|
|
|
const siteConfig = PATTERN_DATABASE[siteKey];
|
|
const pattern = siteConfig?.patterns?.[intentHint];
|
|
|
|
if (!pattern) {
|
|
throw new Error(`No pattern found for ${intentHint} on ${siteKey}`);
|
|
}
|
|
|
|
const elements = [];
|
|
for (const [elementType, selector] of Object.entries(pattern)) {
|
|
if (elementType === "confidence") continue;
|
|
|
|
const element = document.querySelector(selector);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: elementType,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence || 0.8,
|
|
});
|
|
}
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: pattern.confidence || 0.8,
|
|
site: siteKey,
|
|
};
|
|
}
|
|
|
|
detectSite(hostname) {
|
|
for (const [siteKey, config] of Object.entries(PATTERN_DATABASE)) {
|
|
if (siteKey === "universal") continue;
|
|
if (
|
|
config.domains?.some(
|
|
(domain) => hostname === domain || hostname.endsWith(`.${domain}`)
|
|
)
|
|
) {
|
|
return siteKey;
|
|
}
|
|
}
|
|
return "universal";
|
|
}
|
|
|
|
getUniversalPattern(intentHint) {
|
|
const universalPatterns = PATTERN_DATABASE.universal;
|
|
const pattern = universalPatterns[intentHint];
|
|
|
|
if (!pattern) {
|
|
throw new Error(`No universal pattern for ${intentHint}`);
|
|
}
|
|
|
|
const elements = [];
|
|
for (const selector of pattern.selectors) {
|
|
const element = document.querySelector(selector);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: intentHint,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence,
|
|
});
|
|
break; // Take first match for universal patterns
|
|
}
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: pattern.confidence,
|
|
site: "universal",
|
|
};
|
|
}
|
|
|
|
async trySemanticAnalysis(intentHint, focusArea) {
|
|
const relevantElements = document.querySelectorAll(`
|
|
button, input, select, textarea, a[href],
|
|
[role="button"], [role="textbox"], [role="searchbox"],
|
|
[aria-label], [data-testid]
|
|
`);
|
|
|
|
const elements = Array.from(relevantElements)
|
|
.filter((el) => this.isVisible(el))
|
|
.slice(0, 20)
|
|
.map((element) => {
|
|
const elementId = this.registerElement(element);
|
|
return {
|
|
id: elementId,
|
|
type: this.inferElementType(element, intentHint),
|
|
selector: this.generateSelector(element),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intentHint),
|
|
};
|
|
})
|
|
.filter((el) => el.confidence > 0.3)
|
|
.sort((a, b) => b.confidence - a.confidence);
|
|
|
|
return {
|
|
elements,
|
|
confidence:
|
|
elements.length > 0
|
|
? Math.max(...elements.map((e) => e.confidence))
|
|
: 0,
|
|
};
|
|
}
|
|
|
|
async extractContent({ content_type, max_items = 20, summarize = true }) {
|
|
const startTime = performance.now();
|
|
const extractors = {
|
|
article: () => this.extractArticleContent(),
|
|
search_results: () => this.extractSearchResults(max_items),
|
|
posts: () => this.extractPosts(max_items),
|
|
};
|
|
|
|
const extractor = extractors[content_type];
|
|
if (!extractor) {
|
|
throw new Error(`Unknown content type: ${content_type}`);
|
|
}
|
|
|
|
const rawContent = extractor();
|
|
|
|
if (summarize) {
|
|
// Return summary instead of full content to save tokens
|
|
return {
|
|
content_type,
|
|
summary: this.summarizeContent(rawContent, content_type),
|
|
items_found: Array.isArray(rawContent) ? rawContent.length : 1,
|
|
sample_items: Array.isArray(rawContent)
|
|
? rawContent.slice(0, 3)
|
|
: [rawContent],
|
|
extraction_method: this.getExtractionMethod(content_type),
|
|
token_estimate: this.estimateContentTokens(rawContent),
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
extracted_at: new Date().toISOString(),
|
|
};
|
|
} else {
|
|
// Legacy full content extraction
|
|
return {
|
|
content: rawContent,
|
|
method: "semantic_extraction",
|
|
content_type: content_type,
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
extracted_at: new Date().toISOString(),
|
|
};
|
|
}
|
|
}
|
|
|
|
extractArticleContent() {
|
|
const article = document.querySelector(
|
|
'article, [role="article"], .article-content, main'
|
|
);
|
|
const title = document
|
|
.querySelector("h1, .article-title, .post-title")
|
|
?.textContent?.trim();
|
|
const content = article?.textContent?.trim() || this.extractMainContent();
|
|
|
|
return {
|
|
title,
|
|
content,
|
|
word_count: content?.split(/\s+/).length || 0,
|
|
};
|
|
}
|
|
|
|
extractMainContent() {
|
|
// Simple heuristic to find main content
|
|
const candidates = document.querySelectorAll(
|
|
"main, .content, .post-content, .article-body"
|
|
);
|
|
let bestCandidate = null;
|
|
let maxTextLength = 0;
|
|
|
|
for (const candidate of candidates) {
|
|
const textLength = candidate.textContent.trim().length;
|
|
if (textLength > maxTextLength) {
|
|
maxTextLength = textLength;
|
|
bestCandidate = candidate;
|
|
}
|
|
}
|
|
|
|
return (
|
|
bestCandidate?.textContent?.trim() || document.body.textContent.trim()
|
|
);
|
|
}
|
|
|
|
extractSearchResults(max_items = 20) {
|
|
// Common search result patterns
|
|
const selectors = [
|
|
'.search-result, .result-item, [data-testid*="result"]',
|
|
".g, .result, .search-item", // Google-style
|
|
'li[data-testid="search-result"], .SearchResult', // Twitter/X
|
|
".Box-row, .issue-list-item", // GitHub
|
|
"article, .post, .entry", // Generic content
|
|
];
|
|
|
|
let results = [];
|
|
for (const selector of selectors) {
|
|
const elements = document.querySelectorAll(selector);
|
|
if (elements.length > 0) {
|
|
results = Array.from(elements)
|
|
.slice(0, max_items)
|
|
.map((el, index) => ({
|
|
index: index + 1,
|
|
title: this.extractResultTitle(el),
|
|
summary: this.extractResultSummary(el),
|
|
link: this.extractResultLink(el),
|
|
type: this.detectResultType(el),
|
|
score: this.scoreSearchResult(el),
|
|
}));
|
|
break;
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
extractPosts(max_items = 20) {
|
|
// Social media post patterns
|
|
const selectors = [
|
|
'[data-testid="tweet"], .tweet, .post',
|
|
'article[role="article"]', // Twitter/X posts
|
|
".timeline-item, .feed-item",
|
|
".status, .update, .entry",
|
|
];
|
|
|
|
let posts = [];
|
|
for (const selector of selectors) {
|
|
const elements = document.querySelectorAll(selector);
|
|
if (elements.length > 0) {
|
|
posts = Array.from(elements)
|
|
.slice(0, max_items)
|
|
.map((el, index) => ({
|
|
index: index + 1,
|
|
text: this.extractPostText(el),
|
|
author: this.extractPostAuthor(el),
|
|
timestamp: this.extractPostTimestamp(el),
|
|
metrics: this.extractPostMetrics(el),
|
|
has_media: this.hasPostMedia(el),
|
|
post_type: this.detectPostType(el),
|
|
}));
|
|
break;
|
|
}
|
|
}
|
|
|
|
return posts;
|
|
}
|
|
|
|
// Content summarization methods
|
|
summarizeContent(content, content_type) {
|
|
switch (content_type) {
|
|
case "article":
|
|
return this.summarizeArticle(content);
|
|
case "search_results":
|
|
return this.summarizeSearchResults(content);
|
|
case "posts":
|
|
return this.summarizePosts(content);
|
|
default:
|
|
return { summary: "Unknown content type" };
|
|
}
|
|
}
|
|
|
|
summarizeArticle(content) {
|
|
return {
|
|
title: content.title || "Untitled",
|
|
word_count: content.word_count || 0,
|
|
reading_time: Math.ceil((content.word_count || 0) / 200),
|
|
has_images: document.querySelectorAll("img").length > 0,
|
|
has_videos:
|
|
document.querySelectorAll(
|
|
'video, iframe[src*="youtube"], iframe[src*="vimeo"]'
|
|
).length > 0,
|
|
preview:
|
|
content.content?.substring(0, 200) +
|
|
(content.content?.length > 200 ? "..." : ""),
|
|
estimated_tokens: Math.ceil((content.content?.length || 0) / 4),
|
|
};
|
|
}
|
|
|
|
summarizeSearchResults(results) {
|
|
const domains = results
|
|
.map((r) => r.link)
|
|
.filter(Boolean)
|
|
.map((url) => {
|
|
try {
|
|
return new URL(url).hostname;
|
|
} catch {
|
|
return null;
|
|
}
|
|
})
|
|
.filter(Boolean);
|
|
|
|
return {
|
|
total_results: results.length,
|
|
result_types: [...new Set(results.map((r) => r.type))],
|
|
top_domains: this.getTopDomains(domains),
|
|
avg_score:
|
|
results.reduce((sum, r) => sum + (r.score || 0), 0) / results.length,
|
|
has_sponsored: results.some((r) => r.type === "sponsored"),
|
|
quality_score: this.calculateQualityScore(results),
|
|
};
|
|
}
|
|
|
|
summarizePosts(posts) {
|
|
const totalTextLength = posts.reduce(
|
|
(sum, p) => sum + (p.text?.length || 0),
|
|
0
|
|
);
|
|
const totalLikes = posts.reduce(
|
|
(sum, p) => sum + (p.metrics?.likes || 0),
|
|
0
|
|
);
|
|
|
|
return {
|
|
post_count: posts.length,
|
|
avg_length: Math.round(totalTextLength / posts.length),
|
|
has_media_count: posts.filter((p) => p.has_media).length,
|
|
engagement_total: totalLikes,
|
|
avg_engagement: Math.round(totalLikes / posts.length),
|
|
post_types: [...new Set(posts.map((p) => p.post_type))],
|
|
authors: [...new Set(posts.map((p) => p.author).filter(Boolean))].length,
|
|
estimated_tokens: Math.ceil(totalTextLength / 4),
|
|
};
|
|
}
|
|
|
|
// Helper methods for extraction
|
|
extractResultTitle(element) {
|
|
const titleSelectors = [
|
|
'h1, h2, h3, .title, .headline, [data-testid*="title"]',
|
|
];
|
|
for (const selector of titleSelectors) {
|
|
const title = element.querySelector(selector)?.textContent?.trim();
|
|
if (title) return title.substring(0, 100);
|
|
}
|
|
return element.textContent?.trim()?.substring(0, 50) || "No title";
|
|
}
|
|
|
|
extractResultSummary(element) {
|
|
const summarySelectors = [".summary, .description, .snippet, .excerpt"];
|
|
for (const selector of summarySelectors) {
|
|
const summary = element.querySelector(selector)?.textContent?.trim();
|
|
if (summary) return summary.substring(0, 200);
|
|
}
|
|
return element.textContent?.trim()?.substring(0, 150) || "";
|
|
}
|
|
|
|
extractResultLink(element) {
|
|
const link =
|
|
element.querySelector("a[href]")?.href ||
|
|
element.closest("a[href]")?.href ||
|
|
element.getAttribute("href");
|
|
return link || null;
|
|
}
|
|
|
|
detectResultType(element) {
|
|
if (
|
|
element.textContent?.toLowerCase().includes("sponsored") ||
|
|
element.querySelector(".ad, .sponsored")
|
|
)
|
|
return "sponsored";
|
|
if (element.querySelector("img, video")) return "media";
|
|
if (element.querySelector(".price, .cost")) return "product";
|
|
return "organic";
|
|
}
|
|
|
|
scoreSearchResult(element) {
|
|
let score = 0.5;
|
|
if (element.querySelector("h1, h2, h3")) score += 0.2;
|
|
if (element.querySelector("img")) score += 0.1;
|
|
if (element.textContent?.length > 100) score += 0.1;
|
|
if (element.querySelector("a[href]")) score += 0.1;
|
|
return Math.min(score, 1.0);
|
|
}
|
|
|
|
extractPostText(element) {
|
|
const textSelectors = [
|
|
'[data-testid="tweetText"], .tweet-text',
|
|
".post-content, .entry-content",
|
|
".status-content, .message-content",
|
|
];
|
|
|
|
for (const selector of textSelectors) {
|
|
const text = element.querySelector(selector)?.textContent?.trim();
|
|
if (text) return text.substring(0, 280);
|
|
}
|
|
|
|
return element.textContent?.trim()?.substring(0, 280) || "";
|
|
}
|
|
|
|
extractPostAuthor(element) {
|
|
const authorSelectors = [
|
|
'[data-testid="User-Name"], .username',
|
|
".author, .user-name, .handle",
|
|
];
|
|
|
|
for (const selector of authorSelectors) {
|
|
const author = element.querySelector(selector)?.textContent?.trim();
|
|
if (author) return author.substring(0, 50);
|
|
}
|
|
return "Unknown";
|
|
}
|
|
|
|
extractPostTimestamp(element) {
|
|
const timeSelectors = ['time, .timestamp, .date, [data-testid*="time"]'];
|
|
for (const selector of timeSelectors) {
|
|
const time = element.querySelector(selector);
|
|
if (time) {
|
|
return (
|
|
time.getAttribute("datetime") || time.textContent?.trim() || null
|
|
);
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
extractPostMetrics(element) {
|
|
const metrics = {};
|
|
const likeSelectors = ['[data-testid*="like"], .like-count, .heart-count'];
|
|
const replySelectors = [
|
|
'[data-testid*="reply"], .reply-count, .comment-count',
|
|
];
|
|
const shareSelectors = [
|
|
'[data-testid*="retweet"], .share-count, .repost-count',
|
|
];
|
|
|
|
for (const selector of likeSelectors) {
|
|
const likes = element
|
|
.querySelector(selector)
|
|
?.textContent?.match(/\d+/)?.[0];
|
|
if (likes) metrics.likes = parseInt(likes);
|
|
}
|
|
|
|
for (const selector of replySelectors) {
|
|
const replies = element
|
|
.querySelector(selector)
|
|
?.textContent?.match(/\d+/)?.[0];
|
|
if (replies) metrics.replies = parseInt(replies);
|
|
}
|
|
|
|
for (const selector of shareSelectors) {
|
|
const shares = element
|
|
.querySelector(selector)
|
|
?.textContent?.match(/\d+/)?.[0];
|
|
if (shares) metrics.shares = parseInt(shares);
|
|
}
|
|
|
|
return metrics;
|
|
}
|
|
|
|
hasPostMedia(element) {
|
|
return element.querySelector('img, video, [data-testid*="media"]') !== null;
|
|
}
|
|
|
|
detectPostType(element) {
|
|
if (element.querySelector('[data-testid*="retweet"]')) return "repost";
|
|
if (element.querySelector('[data-testid*="reply"]')) return "reply";
|
|
if (element.hasAttribute("data-promoted")) return "promoted";
|
|
return "original";
|
|
}
|
|
|
|
getTopDomains(domains, limit = 5) {
|
|
const domainCounts = {};
|
|
domains.forEach((domain) => {
|
|
domainCounts[domain] = (domainCounts[domain] || 0) + 1;
|
|
});
|
|
|
|
return Object.entries(domainCounts)
|
|
.sort(([, a], [, b]) => b - a)
|
|
.slice(0, limit)
|
|
.map(([domain, count]) => ({ domain, count }));
|
|
}
|
|
|
|
calculateQualityScore(results) {
|
|
const avgScore =
|
|
results.reduce((sum, r) => sum + (r.score || 0), 0) / results.length;
|
|
const hasLinks = results.filter((r) => r.link).length / results.length;
|
|
const hasContent =
|
|
results.filter((r) => r.summary?.length > 50).length / results.length;
|
|
|
|
return Math.round(
|
|
(avgScore * 0.4 + hasLinks * 0.3 + hasContent * 0.3) * 100
|
|
);
|
|
}
|
|
|
|
getExtractionMethod(content_type) {
|
|
const hostname = window.location.hostname;
|
|
if (hostname.includes("twitter") || hostname.includes("x.com"))
|
|
return "twitter_patterns";
|
|
if (hostname.includes("github")) return "github_patterns";
|
|
if (hostname.includes("google")) return "google_patterns";
|
|
return `semantic_${content_type}`;
|
|
}
|
|
|
|
estimateContentTokens(content) {
|
|
if (Array.isArray(content)) {
|
|
return content.reduce((sum, item) => {
|
|
return sum + Math.ceil(JSON.stringify(item).length / 4);
|
|
}, 0);
|
|
} else {
|
|
return Math.ceil(JSON.stringify(content).length / 4);
|
|
}
|
|
}
|
|
|
|
async clickElement({ element_id, click_type = "left", wait_after = 500 }) {
|
|
const element = this.getElementById(element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${element_id}`);
|
|
}
|
|
|
|
// Scroll element into view
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
|
|
// Click the element
|
|
if (click_type === "right") {
|
|
element.dispatchEvent(new MouseEvent("contextmenu", { bubbles: true }));
|
|
} else {
|
|
element.click();
|
|
}
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, wait_after));
|
|
|
|
return {
|
|
success: true,
|
|
element_id,
|
|
click_type,
|
|
element_name: this.getElementName(element),
|
|
};
|
|
}
|
|
|
|
async ensureProperFocus(element) {
|
|
// Scroll element into view first
|
|
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
|
|
// Simulate proper mouse interaction sequence
|
|
const rect = element.getBoundingClientRect();
|
|
const centerX = rect.left + rect.width / 2;
|
|
const centerY = rect.top + rect.height / 2;
|
|
|
|
// Fire mouse events in sequence
|
|
element.dispatchEvent(
|
|
new MouseEvent("mousedown", {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY,
|
|
})
|
|
);
|
|
|
|
element.dispatchEvent(
|
|
new MouseEvent("mouseup", {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY,
|
|
})
|
|
);
|
|
|
|
element.dispatchEvent(
|
|
new MouseEvent("click", {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY,
|
|
})
|
|
);
|
|
|
|
// Focus and fire focus events
|
|
element.focus();
|
|
element.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
|
element.dispatchEvent(new FocusEvent("focus", { bubbles: true }));
|
|
|
|
// Wait for React/framework to update
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
|
|
async clearElementContent(element) {
|
|
if (element.tagName === "INPUT" || element.tagName === "TEXTAREA") {
|
|
element.value = "";
|
|
element.dispatchEvent(new Event("input", { bubbles: true }));
|
|
} else if (element.contentEditable === "true") {
|
|
// For contenteditable, simulate selecting all and deleting
|
|
element.focus();
|
|
document.execCommand("selectAll");
|
|
document.execCommand("delete");
|
|
element.dispatchEvent(new Event("input", { bubbles: true }));
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
|
|
async fillWithEvents(element, value) {
|
|
if (element.tagName === "INPUT" || element.tagName === "TEXTAREA") {
|
|
// Set value and fire comprehensive events
|
|
element.value = value;
|
|
element.dispatchEvent(new Event("beforeinput", { bubbles: true }));
|
|
element.dispatchEvent(new Event("input", { bubbles: true }));
|
|
element.dispatchEvent(new Event("change", { bubbles: true }));
|
|
|
|
// Fire keyboard events to simulate typing completion
|
|
element.dispatchEvent(
|
|
new KeyboardEvent("keydown", { bubbles: true, key: "End" })
|
|
);
|
|
element.dispatchEvent(
|
|
new KeyboardEvent("keyup", { bubbles: true, key: "End" })
|
|
);
|
|
} else if (element.contentEditable === "true") {
|
|
// For contenteditable elements (like Twitter)
|
|
element.textContent = value;
|
|
element.dispatchEvent(new Event("beforeinput", { bubbles: true }));
|
|
element.dispatchEvent(new Event("input", { bubbles: true }));
|
|
|
|
// Trigger composition events for better compatibility
|
|
element.dispatchEvent(
|
|
new CompositionEvent("compositionend", {
|
|
bubbles: true,
|
|
data: value,
|
|
})
|
|
);
|
|
|
|
// Fire selection change to notify frameworks
|
|
document.dispatchEvent(new Event("selectionchange"));
|
|
}
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
|
|
getElementValue(element) {
|
|
if (element.tagName === "INPUT" || element.tagName === "TEXTAREA") {
|
|
return element.value;
|
|
} else if (element.contentEditable === "true") {
|
|
return element.textContent || element.innerText || "";
|
|
}
|
|
return "";
|
|
}
|
|
|
|
// Element state detection methods
|
|
getElementState(element) {
|
|
const state = {
|
|
disabled: this.isElementDisabled(element),
|
|
visible: this.isLikelyVisible(element),
|
|
clickable: this.isElementClickable(element),
|
|
focusable: this.isElementFocusable(element),
|
|
hasText: this.hasText(element),
|
|
isEmpty: this.isEmpty(element),
|
|
};
|
|
|
|
// Overall interaction readiness
|
|
state.interaction_ready =
|
|
state.visible && !state.disabled && (state.clickable || state.focusable);
|
|
|
|
return state;
|
|
}
|
|
|
|
isElementDisabled(element) {
|
|
// Check disabled attribute
|
|
if (element.disabled === true) return true;
|
|
if (element.getAttribute("disabled") !== null) return true;
|
|
|
|
// Check aria-disabled
|
|
if (element.getAttribute("aria-disabled") === "true") return true;
|
|
|
|
// Check common disabled classes
|
|
const disabledClasses = [
|
|
"disabled",
|
|
"btn-disabled",
|
|
"button-disabled",
|
|
"inactive",
|
|
];
|
|
const classList = Array.from(element.classList);
|
|
if (disabledClasses.some((cls) => classList.includes(cls))) return true;
|
|
|
|
// Check if parent form/fieldset is disabled
|
|
const parentFieldset = element.closest("fieldset[disabled]");
|
|
if (parentFieldset) return true;
|
|
|
|
// Check computed styles for pointer-events: none
|
|
const computedStyle = getComputedStyle(element);
|
|
if (computedStyle.pointerEvents === "none") return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
isElementClickable(element) {
|
|
const clickableTags = ["BUTTON", "A", "INPUT"];
|
|
const clickableTypes = ["button", "submit", "reset"];
|
|
const clickableRoles = ["button", "link", "menuitem", "tab"];
|
|
|
|
// Check tag and type
|
|
if (clickableTags.includes(element.tagName)) return true;
|
|
if (element.type && clickableTypes.includes(element.type)) return true;
|
|
|
|
// Check role
|
|
const role = element.getAttribute("role");
|
|
if (role && clickableRoles.includes(role)) return true;
|
|
|
|
// Check for click handlers
|
|
if (element.onclick || element.getAttribute("onclick")) return true;
|
|
|
|
// Check for common clickable classes
|
|
const clickableClasses = ["btn", "button", "clickable", "link"];
|
|
const classList = Array.from(element.classList);
|
|
if (clickableClasses.some((cls) => classList.includes(cls))) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
isElementFocusable(element) {
|
|
const focusableTags = ["INPUT", "TEXTAREA", "SELECT", "BUTTON", "A"];
|
|
|
|
// Check if element is naturally focusable
|
|
if (focusableTags.includes(element.tagName)) return true;
|
|
|
|
// Check tabindex
|
|
const tabindex = element.getAttribute("tabindex");
|
|
if (tabindex && tabindex !== "-1") return true;
|
|
|
|
// Check contenteditable
|
|
if (element.contentEditable === "true") return true;
|
|
|
|
// Check role
|
|
const focusableRoles = ["textbox", "searchbox", "button", "link"];
|
|
const role = element.getAttribute("role");
|
|
if (role && focusableRoles.includes(role)) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
hasText(element) {
|
|
const text =
|
|
element.textContent ||
|
|
element.value ||
|
|
element.getAttribute("aria-label") ||
|
|
"";
|
|
return text.trim().length > 0;
|
|
}
|
|
|
|
isEmpty(element) {
|
|
if (element.tagName === "INPUT" || element.tagName === "TEXTAREA") {
|
|
return !element.value || element.value.trim().length === 0;
|
|
}
|
|
if (element.contentEditable === "true") {
|
|
return !element.textContent || element.textContent.trim().length === 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async waitForCondition({ condition_type, selector, text, timeout = 5000 }) {
|
|
const startTime = Date.now();
|
|
|
|
const conditions = {
|
|
element_visible: () => {
|
|
const el = document.querySelector(selector);
|
|
return el && el.offsetParent !== null;
|
|
},
|
|
text_present: () => document.body.textContent.includes(text),
|
|
};
|
|
|
|
const checkCondition = conditions[condition_type];
|
|
if (!checkCondition) {
|
|
throw new Error(`Unknown condition type: ${condition_type}`);
|
|
}
|
|
|
|
while (Date.now() - startTime < timeout) {
|
|
if (checkCondition()) {
|
|
return {
|
|
condition_met: true,
|
|
wait_time: Date.now() - startTime,
|
|
};
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
|
|
throw new Error(`Timeout waiting for condition: ${condition_type}`);
|
|
}
|
|
|
|
// Utility methods
|
|
registerElement(element) {
|
|
const id = `element_${++this.idCounter}`;
|
|
this.elementRegistry.set(id, element);
|
|
return id;
|
|
}
|
|
|
|
getElementById(id) {
|
|
// Check quick registry first (for q1, q2, etc.)
|
|
if (id.startsWith("q")) {
|
|
return this.quickRegistry.get(id);
|
|
}
|
|
// Then check main registry (for element_1, element_2, etc.)
|
|
return this.elementRegistry.get(id);
|
|
}
|
|
|
|
getElementName(element) {
|
|
return (
|
|
element.getAttribute("aria-label") ||
|
|
element.getAttribute("title") ||
|
|
element.textContent?.trim()?.substring(0, 50) ||
|
|
element.placeholder ||
|
|
element.tagName.toLowerCase()
|
|
);
|
|
}
|
|
|
|
isVisible(element) {
|
|
return (
|
|
element.offsetParent !== null &&
|
|
getComputedStyle(element).visibility !== "hidden" &&
|
|
getComputedStyle(element).opacity !== "0"
|
|
);
|
|
}
|
|
|
|
generateSelector(element) {
|
|
if (element.id) return `#${element.id}`;
|
|
if (element.getAttribute("data-testid"))
|
|
return `[data-testid="${element.getAttribute("data-testid")}"]`;
|
|
|
|
let selector = element.tagName.toLowerCase();
|
|
if (element.className) {
|
|
selector += `.${element.className.split(" ").join(".")}`;
|
|
}
|
|
return selector;
|
|
}
|
|
|
|
inferElementType(element, intentHint) {
|
|
const tagName = element.tagName.toLowerCase();
|
|
const role = element.getAttribute("role");
|
|
const type = element.getAttribute("type");
|
|
|
|
if (tagName === "input" && type === "search") return "search_input";
|
|
if (tagName === "input") return "input";
|
|
if (tagName === "textarea") return "textarea";
|
|
if (tagName === "button" || role === "button") return "button";
|
|
if (tagName === "a") return "link";
|
|
|
|
return "element";
|
|
}
|
|
|
|
calculateConfidence(element, intentHint) {
|
|
let confidence = 0.5;
|
|
|
|
const text = this.getElementName(element).toLowerCase();
|
|
const hint = intentHint.toLowerCase();
|
|
|
|
if (text.includes(hint)) confidence += 0.3;
|
|
if (element.getAttribute("data-testid")) confidence += 0.2;
|
|
if (element.getAttribute("aria-label")) confidence += 0.1;
|
|
|
|
return Math.min(confidence, 1.0);
|
|
}
|
|
|
|
formatAnalysisResult(result, method, startTime) {
|
|
return {
|
|
...result,
|
|
method,
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
analyzed_at: new Date().toISOString(),
|
|
};
|
|
}
|
|
|
|
// Two-phase utility methods
|
|
compressElement(element, isQuick = false) {
|
|
const actualElement = element.element || element;
|
|
const state = this.getElementState(actualElement);
|
|
|
|
if (isQuick) {
|
|
// Quick phase - minimal data with state
|
|
const quickId = `q${++this.quickIdCounter}`;
|
|
this.quickRegistry.set(quickId, actualElement);
|
|
|
|
return {
|
|
id: quickId,
|
|
type: element.type || "element",
|
|
name: element.name?.substring(0, 20) || "unnamed",
|
|
conf: Math.round((element.confidence || 0.5) * 100),
|
|
selector: element.selector || "unknown",
|
|
state: state.disabled ? "disabled" : "enabled",
|
|
clickable: state.clickable,
|
|
ready: state.interaction_ready,
|
|
};
|
|
} else {
|
|
// Detailed phase - compact fingerprint with full state
|
|
return {
|
|
id: element.id,
|
|
fp: this.generateFingerprint(actualElement),
|
|
name: element.name?.substring(0, 30) || "unnamed",
|
|
conf: Math.round((element.confidence || 0.5) * 100),
|
|
meta: {
|
|
...this.getElementMeta(actualElement),
|
|
state: state,
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
generateFingerprint(element) {
|
|
const tag = element.tagName.toLowerCase();
|
|
const primaryClass = this.getPrimaryClass(element);
|
|
const context = this.getContext(element);
|
|
const position = this.getRelativePosition(element);
|
|
|
|
return `${tag}${
|
|
primaryClass ? "." + primaryClass : ""
|
|
}@${context}.${position}`;
|
|
}
|
|
|
|
getPrimaryClass(element) {
|
|
const importantClasses = [
|
|
"btn",
|
|
"button",
|
|
"link",
|
|
"input",
|
|
"search",
|
|
"submit",
|
|
"primary",
|
|
"secondary",
|
|
];
|
|
const classList = Array.from(element.classList);
|
|
return (
|
|
classList.find((cls) => importantClasses.includes(cls)) || classList[0]
|
|
);
|
|
}
|
|
|
|
getContext(element) {
|
|
const parent =
|
|
element.closest("nav, main, header, footer, form, section, article") ||
|
|
element.parentElement;
|
|
if (!parent) return "body";
|
|
return parent.tagName.toLowerCase();
|
|
}
|
|
|
|
getRelativePosition(element) {
|
|
const siblings = Array.from(element.parentElement?.children || []);
|
|
const sameTypeElements = siblings.filter(
|
|
(el) => el.tagName === element.tagName
|
|
);
|
|
return sameTypeElements.indexOf(element) + 1;
|
|
}
|
|
|
|
getElementMeta(element) {
|
|
const rect = element.getBoundingClientRect();
|
|
return {
|
|
rect: [
|
|
Math.round(rect.x),
|
|
Math.round(rect.y),
|
|
Math.round(rect.width),
|
|
Math.round(rect.height),
|
|
],
|
|
visible: this.isLikelyVisible(element),
|
|
form_context: element.closest("form") ? "form" : null,
|
|
};
|
|
}
|
|
|
|
detectPageType() {
|
|
const hostname = window.location.hostname;
|
|
const title = document.title.toLowerCase();
|
|
const hasSearch = document.querySelector(
|
|
'[type="search"], [role="searchbox"]'
|
|
);
|
|
const hasLogin = document.querySelector(
|
|
'[type="password"], [name*="login" i]'
|
|
);
|
|
const hasPost = document.querySelector(
|
|
'[contenteditable="true"], textarea[placeholder*="post" i]'
|
|
);
|
|
|
|
if (hostname.includes("twitter") || hostname.includes("x.com"))
|
|
return "social_media";
|
|
if (hostname.includes("github")) return "code_repository";
|
|
if (hostname.includes("google")) return "search_engine";
|
|
if (hasPost) return "content_creation";
|
|
if (hasLogin) return "authentication";
|
|
if (hasSearch) return "search_interface";
|
|
if (title.includes("shop") || title.includes("store")) return "ecommerce";
|
|
|
|
return "general_website";
|
|
}
|
|
|
|
countViewportElements() {
|
|
const elements = document.querySelectorAll(
|
|
"button, input, select, textarea, a[href]"
|
|
);
|
|
const viewportElements = Array.from(elements).filter((el) =>
|
|
this.isLikelyVisible(el)
|
|
);
|
|
|
|
return {
|
|
buttons: viewportElements.filter(
|
|
(el) => el.tagName === "BUTTON" || el.getAttribute("role") === "button"
|
|
).length,
|
|
inputs: viewportElements.filter((el) => el.tagName === "INPUT").length,
|
|
links: viewportElements.filter((el) => el.tagName === "A").length,
|
|
textareas: viewportElements.filter((el) => el.tagName === "TEXTAREA")
|
|
.length,
|
|
selects: viewportElements.filter((el) => el.tagName === "SELECT").length,
|
|
};
|
|
}
|
|
|
|
async quickViewportScan(intent_hint, maxResults = 3) {
|
|
const candidates = document.querySelectorAll(
|
|
'button, input, a[href], [role="button"], textarea'
|
|
);
|
|
const visibleElements = Array.from(candidates)
|
|
.filter((el) => this.isLikelyVisible(el))
|
|
.slice(0, 10); // Limit scan to first 10 visible elements
|
|
|
|
const scoredElements = visibleElements.map((element) => {
|
|
const confidence = this.calculateConfidence(element, intent_hint);
|
|
return {
|
|
element,
|
|
type: this.inferElementType(element, intent_hint),
|
|
name: this.getElementName(element),
|
|
confidence,
|
|
};
|
|
});
|
|
|
|
return scoredElements
|
|
.filter((el) => el.confidence > 0.3)
|
|
.sort((a, b) => b.confidence - a.confidence)
|
|
.slice(0, maxResults)
|
|
.map((el) => this.compressElement(el, true));
|
|
}
|
|
|
|
scoreIntentMatch(intent_hint, quickMatches) {
|
|
if (quickMatches.length === 0) return "none";
|
|
const avgConfidence =
|
|
quickMatches.reduce((sum, match) => sum + match.conf, 0) /
|
|
quickMatches.length;
|
|
|
|
if (avgConfidence >= 80) return "high";
|
|
if (avgConfidence >= 60) return "medium";
|
|
if (avgConfidence >= 40) return "low";
|
|
return "none";
|
|
}
|
|
|
|
suggestPhase2Areas(quickMatches, intent_hint) {
|
|
const suggestions = [];
|
|
const elementTypes = [...new Set(quickMatches.map((m) => m.type))];
|
|
|
|
if (elementTypes.includes("button")) suggestions.push("buttons");
|
|
if (elementTypes.includes("input") || elementTypes.includes("textarea"))
|
|
suggestions.push("forms");
|
|
if (elementTypes.includes("link")) suggestions.push("navigation");
|
|
|
|
// Intent-based suggestions
|
|
if (
|
|
intent_hint.toLowerCase().includes("search") &&
|
|
!suggestions.includes("forms")
|
|
) {
|
|
suggestions.push("search_elements");
|
|
}
|
|
|
|
return suggestions.slice(0, 3);
|
|
}
|
|
|
|
estimatePhase2Tokens(quickMatches) {
|
|
// Estimate tokens needed for detailed analysis
|
|
const baseTokens = 50; // Base overhead
|
|
const tokensPerElement = 15; // Detailed element info
|
|
const contextTokens = 20; // Page context
|
|
|
|
return baseTokens + quickMatches.length * tokensPerElement + contextTokens;
|
|
}
|
|
|
|
async expandQuickMatches(element_ids) {
|
|
const elements = [];
|
|
for (const id of element_ids) {
|
|
const element = this.quickRegistry.get(id);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, ""),
|
|
name: this.getElementName(element),
|
|
confidence: 0.8, // Default confidence for expanded elements
|
|
element: element,
|
|
});
|
|
}
|
|
}
|
|
return elements;
|
|
}
|
|
|
|
async analyzeFocusAreas(focus_areas, intent_hint) {
|
|
const elements = [];
|
|
const areaSelectors = {
|
|
buttons: 'button, [role="button"], input[type="submit"]',
|
|
forms: 'input, textarea, select, [contenteditable="true"]',
|
|
navigation: 'nav a, .nav-item, [role="navigation"] a',
|
|
search_elements:
|
|
'[type="search"], [role="searchbox"], [placeholder*="search" i]',
|
|
};
|
|
|
|
for (const area of focus_areas) {
|
|
const selector = areaSelectors[area];
|
|
if (selector) {
|
|
const areaElements = document.querySelectorAll(selector);
|
|
for (const element of Array.from(areaElements).slice(0, 5)) {
|
|
if (this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intent_hint),
|
|
element: element,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return elements;
|
|
}
|
|
|
|
async fullEnhancedAnalysis(intent_hint, max_results) {
|
|
// Enhanced version of semantic analysis with better filtering
|
|
const relevantElements = document.querySelectorAll(`
|
|
button, input, select, textarea, a[href],
|
|
[role="button"], [role="textbox"], [role="searchbox"],
|
|
[aria-label], [data-testid], [contenteditable="true"]
|
|
`);
|
|
|
|
const elements = Array.from(relevantElements)
|
|
.filter((el) => this.isLikelyVisible(el))
|
|
.slice(0, 30) // Analyze more elements than before
|
|
.map((element) => {
|
|
const elementId = this.registerElement(element);
|
|
return {
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
selector: this.generateSelector(element),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intent_hint),
|
|
element: element,
|
|
};
|
|
})
|
|
.filter((el) => el.confidence > 0.2) // Lower threshold for detailed analysis
|
|
.sort((a, b) => b.confidence - a.confidence);
|
|
|
|
return elements.slice(0, max_results);
|
|
}
|
|
|
|
deduplicateElements(elements) {
|
|
const seen = new Set();
|
|
return elements.filter((element) => {
|
|
const key = element.name + element.type;
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
async enhanceElementMetadata(elements) {
|
|
return elements.map((element) => ({
|
|
...element,
|
|
meta: this.getElementMeta(element.element),
|
|
}));
|
|
}
|
|
|
|
isLikelyVisible(element) {
|
|
const rect = element.getBoundingClientRect();
|
|
const style = getComputedStyle(element);
|
|
|
|
return (
|
|
rect.top < window.innerHeight &&
|
|
rect.bottom > 0 &&
|
|
rect.left < window.innerWidth &&
|
|
rect.right > 0 &&
|
|
style.visibility !== "hidden" &&
|
|
style.opacity !== "0" &&
|
|
style.display !== "none"
|
|
);
|
|
}
|
|
|
|
estimateTokenUsage(result) {
|
|
// Estimate token count based on result size
|
|
const jsonString = JSON.stringify(result);
|
|
return Math.ceil(jsonString.length / 4); // Rough estimate: 4 chars per token
|
|
}
|
|
}
|
|
|
|
// Initialize the automation system
|
|
const browserAutomation = new BrowserAutomation();
|