mirror of
https://github.com/aaronjmars/opendia.git
synced 2025-12-29 16:16:00 +00:00
1804 lines
60 KiB
JavaScript
1804 lines
60 KiB
JavaScript
// Enhanced Browser Automation Content Script
|
|
console.log('OpenDia enhanced content script loaded');
|
|
|
|
// Enhanced Pattern Database with Intent Categories
|
|
const ENHANCED_PATTERNS = {
|
|
// Authentication patterns
|
|
"auth": {
|
|
"login": {
|
|
input: ["[type='email']", "[name*='username' i]", "[placeholder*='email' i]", "[name*='login' i]"],
|
|
password: ["[type='password']", "[name*='password' i]"],
|
|
submit: ["[type='submit']", "button[form]", ".login-btn", "[aria-label*='login' i]"],
|
|
confidence: 0.9
|
|
},
|
|
"signup": {
|
|
input: ["[name*='register' i]", "[placeholder*='signup' i]", "[name*='email' i]"],
|
|
submit: ["[href*='signup']", ".signup-btn", "[aria-label*='register' i]"],
|
|
confidence: 0.85
|
|
}
|
|
},
|
|
|
|
// Content creation patterns
|
|
"content": {
|
|
"post_create": {
|
|
textarea: ["[contenteditable='true']", "textarea[placeholder*='post' i]", "[data-text='true']"],
|
|
submit: ["[data-testid*='post']", ".post-btn", ".publish-btn", "[aria-label*='post' i]"],
|
|
confidence: 0.9
|
|
},
|
|
"comment": {
|
|
textarea: ["textarea[placeholder*='comment' i]", "[role='textbox']", "[placeholder*='reply' i]"],
|
|
submit: [".comment-btn", "[aria-label*='comment' i]", "[aria-label*='reply' i]"],
|
|
confidence: 0.8
|
|
}
|
|
},
|
|
|
|
// Search patterns
|
|
"search": {
|
|
"global": {
|
|
input: ["[type='search']", "[role='searchbox']", "[placeholder*='search' i]", "[name*='search' i]"],
|
|
submit: ["[aria-label*='search' i]", ".search-btn", "button[type='submit']"],
|
|
confidence: 0.85
|
|
}
|
|
},
|
|
|
|
// Navigation patterns
|
|
"nav": {
|
|
"menu": {
|
|
toggle: ["[aria-label*='menu' i]", ".menu-btn", ".hamburger", "[data-toggle='menu']"],
|
|
items: ["nav a", ".nav-item", "[role='menuitem']"],
|
|
confidence: 0.8
|
|
}
|
|
},
|
|
|
|
// Form patterns
|
|
"form": {
|
|
"submit": {
|
|
button: ["[type='submit']", "button[form]", ".submit-btn", "[aria-label*='submit' i]"],
|
|
confidence: 0.85
|
|
},
|
|
"reset": {
|
|
button: ["[type='reset']", ".reset-btn", "[aria-label*='reset' i]"],
|
|
confidence: 0.8
|
|
}
|
|
}
|
|
};
|
|
|
|
// Legacy pattern database for backward compatibility
|
|
const PATTERN_DATABASE = {
|
|
"twitter": {
|
|
"domains": ["twitter.com", "x.com"],
|
|
"patterns": {
|
|
"post_tweet": {
|
|
textarea: "[data-testid='tweetTextarea_0'], [contenteditable='true'][data-text='true']",
|
|
submit: "[data-testid='tweetButtonInline'], [data-testid='tweetButton']",
|
|
confidence: 0.95
|
|
},
|
|
"search": {
|
|
input: "[data-testid='SearchBox_Search_Input'], input[placeholder*='search' i]",
|
|
submit: "[data-testid='SearchBox_Search_Button']",
|
|
confidence: 0.90
|
|
}
|
|
}
|
|
},
|
|
"github": {
|
|
"domains": ["github.com"],
|
|
"patterns": {
|
|
"search": {
|
|
input: "input[placeholder*='Search' i].form-control",
|
|
submit: "button[type='submit']",
|
|
confidence: 0.85
|
|
}
|
|
}
|
|
},
|
|
"universal": {
|
|
"search": {
|
|
selectors: [
|
|
"input[type='search']",
|
|
"input[placeholder*='search' i]",
|
|
"[role='searchbox']",
|
|
"input[name*='search' i]"
|
|
],
|
|
confidence: 0.60
|
|
},
|
|
"submit": {
|
|
selectors: [
|
|
"button[type='submit']:not([disabled])",
|
|
"input[type='submit']:not([disabled])",
|
|
"[role='button'][aria-label*='submit' i]"
|
|
],
|
|
confidence: 0.65
|
|
}
|
|
}
|
|
};
|
|
|
|
// Token usage tracking and performance optimization
|
|
class TokenOptimizer {
|
|
constructor() {
|
|
this.metrics = JSON.parse(localStorage.getItem('opendia_metrics') || '[]');
|
|
this.successRates = JSON.parse(localStorage.getItem('opendia_success_rates') || '{}');
|
|
this.recommendedMaxResults = 5;
|
|
this.lastCleanup = Date.now();
|
|
}
|
|
|
|
trackUsage(operation, tokenCount, success, pageType = 'unknown', method = 'unknown') {
|
|
const metric = {
|
|
operation,
|
|
tokens: tokenCount,
|
|
success,
|
|
pageType,
|
|
method,
|
|
timestamp: Date.now()
|
|
};
|
|
|
|
this.metrics.push(metric);
|
|
|
|
// Keep only last 100 metrics to prevent storage bloat
|
|
if (this.metrics.length > 100) {
|
|
this.metrics = this.metrics.slice(-100);
|
|
}
|
|
|
|
// Auto-adjust limits based on success rates
|
|
this.adjustLimits();
|
|
|
|
// Periodic cleanup (once per hour)
|
|
if (Date.now() - this.lastCleanup > 3600000) {
|
|
this.cleanup();
|
|
}
|
|
|
|
// Persist to localStorage
|
|
localStorage.setItem('opendia_metrics', JSON.stringify(this.metrics));
|
|
}
|
|
|
|
adjustLimits() {
|
|
const recent = this.metrics.slice(-20);
|
|
if (recent.length < 10) return;
|
|
|
|
const avgTokens = recent.reduce((sum, m) => sum + m.tokens, 0) / recent.length;
|
|
const successRate = recent.filter(m => m.success).length / recent.length;
|
|
|
|
// If using too many tokens but success rate is high, reduce max results
|
|
if (avgTokens > 200 && successRate > 0.8) {
|
|
this.recommendedMaxResults = Math.max(3, this.recommendedMaxResults - 1);
|
|
}
|
|
// If success rate is low, increase max results to get better matches
|
|
else if (successRate < 0.6) {
|
|
this.recommendedMaxResults = Math.min(10, this.recommendedMaxResults + 1);
|
|
}
|
|
}
|
|
|
|
cleanup() {
|
|
// Remove metrics older than 7 days
|
|
const cutoff = Date.now() - (7 * 24 * 60 * 60 * 1000);
|
|
this.metrics = this.metrics.filter(m => m.timestamp > cutoff);
|
|
|
|
// Clean up success rates for rarely used combinations
|
|
const recentPageTypes = new Set(this.metrics.map(m => m.pageType));
|
|
Object.keys(this.successRates).forEach(key => {
|
|
const [pageType] = key.split(':');
|
|
if (!recentPageTypes.has(pageType)) {
|
|
delete this.successRates[key];
|
|
}
|
|
});
|
|
|
|
localStorage.setItem('opendia_metrics', JSON.stringify(this.metrics));
|
|
localStorage.setItem('opendia_success_rates', JSON.stringify(this.successRates));
|
|
this.lastCleanup = Date.now();
|
|
}
|
|
|
|
trackMethodSuccess(pageType, intent, method, success) {
|
|
const key = `${pageType}:${intent}:${method}`;
|
|
if (!this.successRates[key]) {
|
|
this.successRates[key] = { attempts: 0, successes: 0 };
|
|
}
|
|
|
|
this.successRates[key].attempts++;
|
|
if (success) {
|
|
this.successRates[key].successes++;
|
|
}
|
|
|
|
localStorage.setItem('opendia_success_rates', JSON.stringify(this.successRates));
|
|
}
|
|
|
|
getBestMethod(pageType, intent) {
|
|
const methods = ['enhanced_pattern_match', 'pattern_database', 'viewport_scan', 'semantic_analysis'];
|
|
|
|
return methods.sort((a, b) => {
|
|
const aKey = `${pageType}:${intent}:${a}`;
|
|
const bKey = `${pageType}:${intent}:${b}`;
|
|
const aRate = this.getSuccessRate(aKey);
|
|
const bRate = this.getSuccessRate(bKey);
|
|
return bRate - aRate;
|
|
})[0];
|
|
}
|
|
|
|
getSuccessRate(key) {
|
|
const data = this.successRates[key];
|
|
if (!data || data.attempts === 0) return 0;
|
|
return data.successes / data.attempts;
|
|
}
|
|
|
|
getRecommendedMaxResults() {
|
|
return this.recommendedMaxResults;
|
|
}
|
|
|
|
getAnalytics() {
|
|
const recent = this.metrics.slice(-50);
|
|
if (recent.length === 0) return { message: 'No metrics available' };
|
|
|
|
const avgTokens = recent.reduce((sum, m) => sum + m.tokens, 0) / recent.length;
|
|
const successRate = recent.filter(m => m.success).length / recent.length;
|
|
const operationBreakdown = {};
|
|
const methodBreakdown = {};
|
|
|
|
recent.forEach(m => {
|
|
operationBreakdown[m.operation] = (operationBreakdown[m.operation] || 0) + 1;
|
|
methodBreakdown[m.method] = (methodBreakdown[m.method] || 0) + 1;
|
|
});
|
|
|
|
return {
|
|
totalOperations: recent.length,
|
|
avgTokensPerOperation: Math.round(avgTokens),
|
|
successRate: Math.round(successRate * 100),
|
|
recommendedMaxResults: this.recommendedMaxResults,
|
|
operationBreakdown,
|
|
methodBreakdown,
|
|
tokenSavings: this.calculateTokenSavings(recent)
|
|
};
|
|
}
|
|
|
|
calculateTokenSavings(metrics) {
|
|
// Estimate token savings vs naive approach
|
|
const actualTokens = metrics.reduce((sum, m) => sum + m.tokens, 0);
|
|
const estimatedNaiveTokens = metrics.length * 300; // Estimate for non-optimized approach
|
|
const savings = estimatedNaiveTokens - actualTokens;
|
|
const percentage = Math.round((savings / estimatedNaiveTokens) * 100);
|
|
|
|
return {
|
|
actualTokens,
|
|
estimatedNaiveTokens,
|
|
tokensSaved: savings,
|
|
percentageSaved: percentage
|
|
};
|
|
}
|
|
}
|
|
|
|
class BrowserAutomation {
|
|
constructor() {
|
|
this.elementRegistry = new Map();
|
|
this.quickRegistry = new Map(); // For phase 1 quick matches
|
|
this.idCounter = 0;
|
|
this.quickIdCounter = 0;
|
|
this.tokenOptimizer = new TokenOptimizer();
|
|
this.setupMessageListener();
|
|
this.setupViewportAnalyzer();
|
|
}
|
|
|
|
setupViewportAnalyzer() {
|
|
this.visibilityMap = new Map();
|
|
this.observer = new IntersectionObserver((entries) => {
|
|
entries.forEach(entry => {
|
|
this.visibilityMap.set(entry.target, {
|
|
visible: entry.isIntersecting,
|
|
ratio: entry.intersectionRatio
|
|
});
|
|
});
|
|
}, { threshold: [0, 0.1, 0.5, 1.0] });
|
|
}
|
|
|
|
setupMessageListener() {
|
|
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
|
|
this.handleMessage(message).then(sendResponse).catch(error => {
|
|
sendResponse({
|
|
success: false,
|
|
error: error.message,
|
|
stack: error.stack
|
|
});
|
|
});
|
|
return true; // Keep message channel open for async response
|
|
});
|
|
}
|
|
|
|
async handleMessage(message) {
|
|
const { action, data } = message;
|
|
const startTime = performance.now();
|
|
|
|
try {
|
|
let result;
|
|
switch (action) {
|
|
case 'analyze':
|
|
result = await this.analyzePage(data);
|
|
break;
|
|
case 'extract_content':
|
|
result = await this.extractContent(data);
|
|
break;
|
|
case 'element_click':
|
|
result = await this.clickElement(data);
|
|
break;
|
|
case 'element_fill':
|
|
result = await this.fillElement(data);
|
|
break;
|
|
case 'wait_for':
|
|
result = await this.waitForCondition(data);
|
|
break;
|
|
case 'getPageInfo':
|
|
result = {
|
|
title: document.title,
|
|
url: window.location.href,
|
|
content: document.body.innerText
|
|
};
|
|
break;
|
|
case 'get_analytics':
|
|
result = this.tokenOptimizer.getAnalytics();
|
|
break;
|
|
case 'clear_analytics':
|
|
localStorage.removeItem('opendia_metrics');
|
|
localStorage.removeItem('opendia_success_rates');
|
|
this.tokenOptimizer = new TokenOptimizer();
|
|
result = { message: 'Analytics cleared successfully' };
|
|
break;
|
|
case 'get_element_state':
|
|
const element = this.getElementById(data.element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${data.element_id}`);
|
|
}
|
|
result = {
|
|
element_id: data.element_id,
|
|
element_name: this.getElementName(element),
|
|
state: this.getElementState(element),
|
|
current_value: this.getElementValue(element)
|
|
};
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown action: ${action}`);
|
|
}
|
|
|
|
const executionTime = performance.now() - startTime;
|
|
const dataSize = new Blob([JSON.stringify(result)]).size;
|
|
|
|
return {
|
|
success: true,
|
|
data: result,
|
|
execution_time: Math.round(executionTime),
|
|
data_size: dataSize,
|
|
timestamp: Date.now()
|
|
};
|
|
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
error: error.message,
|
|
stack: error.stack,
|
|
execution_time: Math.round(performance.now() - startTime)
|
|
};
|
|
}
|
|
}
|
|
|
|
async analyzePage({ intent_hint, phase = 'discover', focus_areas, element_ids, max_results = 5 }) {
|
|
const startTime = performance.now();
|
|
|
|
// Two-phase approach
|
|
if (phase === 'discover') {
|
|
return await this.quickDiscovery({ intent_hint, max_results });
|
|
} else if (phase === 'detailed') {
|
|
return await this.detailedAnalysis({ intent_hint, focus_areas, element_ids, max_results });
|
|
}
|
|
|
|
// Legacy single-phase approach for backward compatibility
|
|
return await this.legacyAnalysis({ intent_hint, focus_area: focus_areas?.[0], max_results });
|
|
}
|
|
|
|
async quickDiscovery({ intent_hint, max_results = 5 }) {
|
|
const startTime = performance.now();
|
|
|
|
// Detect page type and get basic metrics
|
|
const pageType = this.detectPageType();
|
|
const viewportElements = this.countViewportElements();
|
|
|
|
// Use token optimizer recommendations
|
|
const recommendedMaxResults = this.tokenOptimizer.getRecommendedMaxResults();
|
|
max_results = Math.min(max_results, recommendedMaxResults);
|
|
|
|
// Get best method based on historical performance
|
|
const bestMethod = this.tokenOptimizer.getBestMethod(pageType, intent_hint);
|
|
|
|
// Try to find obvious matches using enhanced patterns
|
|
let quickMatches = [];
|
|
let usedMethod = 'quick_discovery';
|
|
|
|
try {
|
|
if (bestMethod === 'enhanced_pattern_match' || bestMethod === 'pattern_database') {
|
|
const patternResult = await this.tryEnhancedPatterns(intent_hint);
|
|
if (patternResult.confidence > 0.7) {
|
|
quickMatches = patternResult.elements.slice(0, 3).map(el => this.compressElement(el, true));
|
|
usedMethod = 'enhanced_patterns';
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn('Enhanced patterns failed:', error);
|
|
}
|
|
|
|
// If no pattern matches, do a quick viewport scan
|
|
if (quickMatches.length === 0) {
|
|
quickMatches = await this.quickViewportScan(intent_hint, 3);
|
|
usedMethod = 'viewport_scan';
|
|
}
|
|
|
|
const intentMatch = this.scoreIntentMatch(intent_hint, quickMatches);
|
|
const suggestedAreas = this.suggestPhase2Areas(quickMatches, intent_hint);
|
|
const executionTime = Math.round(performance.now() - startTime);
|
|
|
|
const result = {
|
|
summary: {
|
|
page_type: pageType,
|
|
intent_match: intentMatch,
|
|
element_count: viewportElements,
|
|
viewport_elements: quickMatches.length,
|
|
suggested_phase2: suggestedAreas
|
|
},
|
|
quick_matches: quickMatches,
|
|
token_estimate: this.estimatePhase2Tokens(quickMatches),
|
|
method: usedMethod,
|
|
execution_time: executionTime,
|
|
intent_hint: intent_hint, // Add this for server.js compatibility
|
|
elements: quickMatches // Add this for backward compatibility
|
|
};
|
|
|
|
// Track token usage and success
|
|
const tokenCount = this.estimateTokenUsage(result);
|
|
const success = quickMatches.length > 0 && intentMatch !== 'none';
|
|
this.tokenOptimizer.trackUsage('page_analyze_discover', tokenCount, success, pageType, usedMethod);
|
|
this.tokenOptimizer.trackMethodSuccess(pageType, intent_hint, usedMethod, success);
|
|
|
|
return result;
|
|
}
|
|
|
|
async detailedAnalysis({ intent_hint, focus_areas, element_ids, max_results = 10 }) {
|
|
const startTime = performance.now();
|
|
const pageType = this.detectPageType();
|
|
|
|
// Use token optimizer recommendations
|
|
const recommendedMaxResults = this.tokenOptimizer.getRecommendedMaxResults();
|
|
max_results = Math.min(max_results, recommendedMaxResults + 2); // Allow slightly more for detailed analysis
|
|
|
|
let elements = [];
|
|
let method = 'detailed_analysis';
|
|
|
|
// Expand specific quick matches if provided
|
|
if (element_ids?.length) {
|
|
elements = await this.expandQuickMatches(element_ids);
|
|
method = 'expanded_matches';
|
|
}
|
|
|
|
// Analyze specific focus areas
|
|
if (focus_areas?.length) {
|
|
const areaElements = await this.analyzeFocusAreas(focus_areas, intent_hint);
|
|
elements = [...elements, ...areaElements];
|
|
method = elements.length > 0 ? 'focus_area_analysis' : method;
|
|
}
|
|
|
|
// If no specific analysis requested, do full enhanced analysis
|
|
if (elements.length === 0) {
|
|
elements = await this.fullEnhancedAnalysis(intent_hint, max_results);
|
|
method = 'full_enhanced_analysis';
|
|
}
|
|
|
|
// Deduplicate and enhance with metadata
|
|
elements = this.deduplicateElements(elements);
|
|
elements = await this.enhanceElementMetadata(elements);
|
|
|
|
// Apply compact fingerprinting
|
|
elements = elements.slice(0, max_results).map(el => this.compressElement(el, false));
|
|
|
|
const executionTime = Math.round(performance.now() - startTime);
|
|
const result = {
|
|
elements,
|
|
interaction_ready: elements.every(el => el.conf > 50),
|
|
method,
|
|
execution_time: executionTime,
|
|
intent_hint: intent_hint // Add this for server.js compatibility
|
|
};
|
|
|
|
// Track token usage and success
|
|
const tokenCount = this.estimateTokenUsage(result);
|
|
const success = elements.length > 0 && elements.some(el => el.conf > 70);
|
|
this.tokenOptimizer.trackUsage('page_analyze_detailed', tokenCount, success, pageType, method);
|
|
this.tokenOptimizer.trackMethodSuccess(pageType, intent_hint, method, success);
|
|
|
|
return result;
|
|
}
|
|
|
|
async legacyAnalysis({ intent_hint, focus_area, max_results = 5 }) {
|
|
const startTime = performance.now();
|
|
let result;
|
|
|
|
try {
|
|
// Try enhanced patterns first
|
|
result = await this.tryEnhancedPatterns(intent_hint);
|
|
if (result.confidence > 0.8) {
|
|
return this.formatAnalysisResult(result, 'enhanced_patterns', startTime);
|
|
}
|
|
} catch (error) {
|
|
console.warn('Enhanced patterns failed, trying legacy patterns:', error);
|
|
try {
|
|
// Fallback to legacy pattern database
|
|
result = await this.tryPatternDatabase(intent_hint);
|
|
if (result.confidence > 0.8) {
|
|
return this.formatAnalysisResult(result, 'pattern_database', startTime);
|
|
}
|
|
} catch (legacyError) {
|
|
console.warn('Legacy pattern database failed:', legacyError);
|
|
}
|
|
}
|
|
|
|
// Final fallback to semantic analysis
|
|
result = await this.trySemanticAnalysis(intent_hint, focus_area);
|
|
return this.formatAnalysisResult(result, 'semantic_analysis', startTime);
|
|
}
|
|
|
|
async tryEnhancedPatterns(intent_hint) {
|
|
const [category, action] = this.parseIntent(intent_hint);
|
|
const pattern = ENHANCED_PATTERNS[category]?.[action];
|
|
|
|
if (!pattern) {
|
|
return this.tryUniversalPatterns(intent_hint);
|
|
}
|
|
|
|
const elements = this.findPatternElements(pattern);
|
|
return {
|
|
elements: elements.slice(0, 3),
|
|
confidence: pattern.confidence,
|
|
method: 'enhanced_pattern_match',
|
|
category,
|
|
action
|
|
};
|
|
}
|
|
|
|
parseIntent(intent) {
|
|
const intentLower = intent.toLowerCase();
|
|
|
|
// Check for authentication patterns
|
|
if (intentLower.includes('login') || intentLower.includes('sign in') || intentLower.includes('log in')) {
|
|
return ['auth', 'login'];
|
|
}
|
|
if (intentLower.includes('signup') || intentLower.includes('sign up') || intentLower.includes('register') || intentLower.includes('create account')) {
|
|
return ['auth', 'signup'];
|
|
}
|
|
|
|
// Check for content creation patterns
|
|
if (intentLower.includes('tweet') || intentLower.includes('post') || intentLower.includes('compose') ||
|
|
intentLower.includes('create') || intentLower.includes('write') || intentLower.includes('publish')) {
|
|
return ['content', 'post_create'];
|
|
}
|
|
if (intentLower.includes('comment') || intentLower.includes('reply')) {
|
|
return ['content', 'comment'];
|
|
}
|
|
|
|
// Check for search patterns
|
|
if (intentLower.includes('search') || intentLower.includes('find') || intentLower.includes('look for')) {
|
|
return ['search', 'global'];
|
|
}
|
|
|
|
// Check for navigation patterns
|
|
if (intentLower.includes('menu') || intentLower.includes('navigation') || intentLower.includes('nav')) {
|
|
return ['nav', 'menu'];
|
|
}
|
|
|
|
// Check for form patterns
|
|
if (intentLower.includes('submit') || intentLower.includes('send') || intentLower.includes('save')) {
|
|
return ['form', 'submit'];
|
|
}
|
|
if (intentLower.includes('reset') || intentLower.includes('clear') || intentLower.includes('cancel')) {
|
|
return ['form', 'reset'];
|
|
}
|
|
|
|
// Fallback - try to infer from context
|
|
if (intentLower.includes('button') || intentLower.includes('click')) {
|
|
return ['form', 'submit'];
|
|
}
|
|
if (intentLower.includes('input') || intentLower.includes('field') || intentLower.includes('text')) {
|
|
return ['content', 'post_create'];
|
|
}
|
|
|
|
// Default fallback
|
|
return ['content', 'post_create']; // More useful default than search
|
|
}
|
|
|
|
findPatternElements(pattern) {
|
|
const elements = [];
|
|
|
|
for (const [elementType, selectors] of Object.entries(pattern)) {
|
|
if (elementType === 'confidence') continue;
|
|
|
|
for (const selector of selectors) {
|
|
const element = document.querySelector(selector);
|
|
if (element && this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: elementType,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence || 0.8,
|
|
element: element
|
|
});
|
|
break; // Take first match per element type
|
|
}
|
|
}
|
|
}
|
|
|
|
return elements;
|
|
}
|
|
|
|
tryUniversalPatterns(intent_hint) {
|
|
const intentLower = intent_hint.toLowerCase();
|
|
let selectors = [];
|
|
|
|
// Content creation patterns
|
|
if (intentLower.includes('tweet') || intentLower.includes('post') || intentLower.includes('compose') ||
|
|
intentLower.includes('create') || intentLower.includes('write')) {
|
|
selectors = [
|
|
"[contenteditable='true']",
|
|
"textarea[placeholder*='tweet' i]",
|
|
"textarea[placeholder*='post' i]",
|
|
"textarea[placeholder*='what' i]",
|
|
"[data-text='true']",
|
|
"[role='textbox']",
|
|
"textarea:not([style*='display: none'])"
|
|
];
|
|
}
|
|
// Authentication patterns
|
|
else if (intentLower.includes('login') || intentLower.includes('sign in')) {
|
|
selectors = [
|
|
"[type='email']",
|
|
"[name*='username' i]",
|
|
"[placeholder*='email' i]",
|
|
"[placeholder*='username' i]",
|
|
"input[name*='login' i]"
|
|
];
|
|
}
|
|
else if (intentLower.includes('signup') || intentLower.includes('register')) {
|
|
selectors = [
|
|
"[href*='signup']",
|
|
".signup-btn",
|
|
"[aria-label*='register' i]",
|
|
"button[data-testid*='signup' i]",
|
|
"a[href*='register']"
|
|
];
|
|
}
|
|
// Search patterns
|
|
else if (intentLower.includes('search') || intentLower.includes('find')) {
|
|
selectors = [
|
|
"[type='search']",
|
|
"[role='searchbox']",
|
|
"[placeholder*='search' i]",
|
|
"[data-testid*='search' i]",
|
|
"input[name*='search' i]"
|
|
];
|
|
}
|
|
// Generic fallback - look for interactive elements
|
|
else {
|
|
selectors = [
|
|
"button:not([disabled])",
|
|
"[contenteditable='true']",
|
|
"textarea",
|
|
"[type='submit']",
|
|
"[role='button']",
|
|
"input[type='text']"
|
|
];
|
|
}
|
|
|
|
const elements = [];
|
|
|
|
for (const selector of selectors) {
|
|
const foundElements = document.querySelectorAll(selector);
|
|
for (const element of foundElements) {
|
|
if (this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: 0.5 + (this.calculateConfidence(element, intent_hint) * 0.3),
|
|
element: element
|
|
});
|
|
if (elements.length >= 3) break; // Limit to 3 elements
|
|
}
|
|
}
|
|
if (elements.length >= 3) break;
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: elements.length > 0 ? Math.max(...elements.map(e => e.confidence)) : 0,
|
|
method: 'universal_pattern'
|
|
};
|
|
}
|
|
|
|
async tryPatternDatabase(intentHint) {
|
|
const hostname = window.location.hostname;
|
|
const siteKey = this.detectSite(hostname);
|
|
|
|
if (siteKey === 'universal') {
|
|
return this.getUniversalPattern(intentHint);
|
|
}
|
|
|
|
const siteConfig = PATTERN_DATABASE[siteKey];
|
|
const pattern = siteConfig?.patterns?.[intentHint];
|
|
|
|
if (!pattern) {
|
|
throw new Error(`No pattern found for ${intentHint} on ${siteKey}`);
|
|
}
|
|
|
|
const elements = [];
|
|
for (const [elementType, selector] of Object.entries(pattern)) {
|
|
if (elementType === 'confidence') continue;
|
|
|
|
const element = document.querySelector(selector);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: elementType,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence || 0.8
|
|
});
|
|
}
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: pattern.confidence || 0.8,
|
|
site: siteKey
|
|
};
|
|
}
|
|
|
|
detectSite(hostname) {
|
|
for (const [siteKey, config] of Object.entries(PATTERN_DATABASE)) {
|
|
if (siteKey === 'universal') continue;
|
|
if (config.domains?.some(domain =>
|
|
hostname === domain || hostname.endsWith(`.${domain}`)
|
|
)) {
|
|
return siteKey;
|
|
}
|
|
}
|
|
return 'universal';
|
|
}
|
|
|
|
getUniversalPattern(intentHint) {
|
|
const universalPatterns = PATTERN_DATABASE.universal;
|
|
const pattern = universalPatterns[intentHint];
|
|
|
|
if (!pattern) {
|
|
throw new Error(`No universal pattern for ${intentHint}`);
|
|
}
|
|
|
|
const elements = [];
|
|
for (const selector of pattern.selectors) {
|
|
const element = document.querySelector(selector);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: intentHint,
|
|
selector: selector,
|
|
name: this.getElementName(element),
|
|
confidence: pattern.confidence
|
|
});
|
|
break; // Take first match for universal patterns
|
|
}
|
|
}
|
|
|
|
return {
|
|
elements,
|
|
confidence: pattern.confidence,
|
|
site: 'universal'
|
|
};
|
|
}
|
|
|
|
async trySemanticAnalysis(intentHint, focusArea) {
|
|
const relevantElements = document.querySelectorAll(`
|
|
button, input, select, textarea, a[href],
|
|
[role="button"], [role="textbox"], [role="searchbox"],
|
|
[aria-label], [data-testid]
|
|
`);
|
|
|
|
const elements = Array.from(relevantElements)
|
|
.filter(el => this.isVisible(el))
|
|
.slice(0, 20)
|
|
.map(element => {
|
|
const elementId = this.registerElement(element);
|
|
return {
|
|
id: elementId,
|
|
type: this.inferElementType(element, intentHint),
|
|
selector: this.generateSelector(element),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intentHint)
|
|
};
|
|
})
|
|
.filter(el => el.confidence > 0.3)
|
|
.sort((a, b) => b.confidence - a.confidence);
|
|
|
|
return {
|
|
elements,
|
|
confidence: elements.length > 0 ? Math.max(...elements.map(e => e.confidence)) : 0
|
|
};
|
|
}
|
|
|
|
async extractContent({ content_type, max_items = 20, summarize = true }) {
|
|
const startTime = performance.now();
|
|
const extractors = {
|
|
'article': () => this.extractArticleContent(),
|
|
'search_results': () => this.extractSearchResults(max_items),
|
|
'posts': () => this.extractPosts(max_items)
|
|
};
|
|
|
|
const extractor = extractors[content_type];
|
|
if (!extractor) {
|
|
throw new Error(`Unknown content type: ${content_type}`);
|
|
}
|
|
|
|
const rawContent = extractor();
|
|
|
|
if (summarize) {
|
|
// Return summary instead of full content to save tokens
|
|
return {
|
|
content_type,
|
|
summary: this.summarizeContent(rawContent, content_type),
|
|
items_found: Array.isArray(rawContent) ? rawContent.length : 1,
|
|
sample_items: Array.isArray(rawContent) ? rawContent.slice(0, 3) : [rawContent],
|
|
extraction_method: this.getExtractionMethod(content_type),
|
|
token_estimate: this.estimateContentTokens(rawContent),
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
extracted_at: new Date().toISOString()
|
|
};
|
|
} else {
|
|
// Legacy full content extraction
|
|
return {
|
|
content: rawContent,
|
|
method: 'semantic_extraction',
|
|
content_type: content_type,
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
extracted_at: new Date().toISOString()
|
|
};
|
|
}
|
|
}
|
|
|
|
extractArticleContent() {
|
|
const article = document.querySelector('article, [role="article"], .article-content, main');
|
|
const title = document.querySelector('h1, .article-title, .post-title')?.textContent?.trim();
|
|
const content = article?.textContent?.trim() || this.extractMainContent();
|
|
|
|
return {
|
|
title,
|
|
content,
|
|
word_count: content?.split(/\s+/).length || 0
|
|
};
|
|
}
|
|
|
|
extractMainContent() {
|
|
// Simple heuristic to find main content
|
|
const candidates = document.querySelectorAll('main, .content, .post-content, .article-body');
|
|
let bestCandidate = null;
|
|
let maxTextLength = 0;
|
|
|
|
for (const candidate of candidates) {
|
|
const textLength = candidate.textContent.trim().length;
|
|
if (textLength > maxTextLength) {
|
|
maxTextLength = textLength;
|
|
bestCandidate = candidate;
|
|
}
|
|
}
|
|
|
|
return bestCandidate?.textContent?.trim() || document.body.textContent.trim();
|
|
}
|
|
|
|
extractSearchResults(max_items = 20) {
|
|
// Common search result patterns
|
|
const selectors = [
|
|
'.search-result, .result-item, [data-testid*="result"]',
|
|
'.g, .result, .search-item', // Google-style
|
|
'li[data-testid="search-result"], .SearchResult', // Twitter/X
|
|
'.Box-row, .issue-list-item', // GitHub
|
|
'article, .post, .entry' // Generic content
|
|
];
|
|
|
|
let results = [];
|
|
for (const selector of selectors) {
|
|
const elements = document.querySelectorAll(selector);
|
|
if (elements.length > 0) {
|
|
results = Array.from(elements).slice(0, max_items).map((el, index) => ({
|
|
index: index + 1,
|
|
title: this.extractResultTitle(el),
|
|
summary: this.extractResultSummary(el),
|
|
link: this.extractResultLink(el),
|
|
type: this.detectResultType(el),
|
|
score: this.scoreSearchResult(el)
|
|
}));
|
|
break;
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
extractPosts(max_items = 20) {
|
|
// Social media post patterns
|
|
const selectors = [
|
|
'[data-testid="tweet"], .tweet, .post',
|
|
'article[role="article"]', // Twitter/X posts
|
|
'.timeline-item, .feed-item',
|
|
'.status, .update, .entry'
|
|
];
|
|
|
|
let posts = [];
|
|
for (const selector of selectors) {
|
|
const elements = document.querySelectorAll(selector);
|
|
if (elements.length > 0) {
|
|
posts = Array.from(elements).slice(0, max_items).map((el, index) => ({
|
|
index: index + 1,
|
|
text: this.extractPostText(el),
|
|
author: this.extractPostAuthor(el),
|
|
timestamp: this.extractPostTimestamp(el),
|
|
metrics: this.extractPostMetrics(el),
|
|
has_media: this.hasPostMedia(el),
|
|
post_type: this.detectPostType(el)
|
|
}));
|
|
break;
|
|
}
|
|
}
|
|
|
|
return posts;
|
|
}
|
|
|
|
// Content summarization methods
|
|
summarizeContent(content, content_type) {
|
|
switch(content_type) {
|
|
case 'article':
|
|
return this.summarizeArticle(content);
|
|
case 'search_results':
|
|
return this.summarizeSearchResults(content);
|
|
case 'posts':
|
|
return this.summarizePosts(content);
|
|
default:
|
|
return { summary: 'Unknown content type' };
|
|
}
|
|
}
|
|
|
|
summarizeArticle(content) {
|
|
return {
|
|
title: content.title || 'Untitled',
|
|
word_count: content.word_count || 0,
|
|
reading_time: Math.ceil((content.word_count || 0) / 200),
|
|
has_images: document.querySelectorAll('img').length > 0,
|
|
has_videos: document.querySelectorAll('video, iframe[src*="youtube"], iframe[src*="vimeo"]').length > 0,
|
|
preview: content.content?.substring(0, 200) + (content.content?.length > 200 ? '...' : ''),
|
|
estimated_tokens: Math.ceil((content.content?.length || 0) / 4)
|
|
};
|
|
}
|
|
|
|
summarizeSearchResults(results) {
|
|
const domains = results.map(r => r.link).filter(Boolean)
|
|
.map(url => {
|
|
try { return new URL(url).hostname; } catch { return null; }
|
|
}).filter(Boolean);
|
|
|
|
return {
|
|
total_results: results.length,
|
|
result_types: [...new Set(results.map(r => r.type))],
|
|
top_domains: this.getTopDomains(domains),
|
|
avg_score: results.reduce((sum, r) => sum + (r.score || 0), 0) / results.length,
|
|
has_sponsored: results.some(r => r.type === 'sponsored'),
|
|
quality_score: this.calculateQualityScore(results)
|
|
};
|
|
}
|
|
|
|
summarizePosts(posts) {
|
|
const totalTextLength = posts.reduce((sum, p) => sum + (p.text?.length || 0), 0);
|
|
const totalLikes = posts.reduce((sum, p) => sum + (p.metrics?.likes || 0), 0);
|
|
|
|
return {
|
|
post_count: posts.length,
|
|
avg_length: Math.round(totalTextLength / posts.length),
|
|
has_media_count: posts.filter(p => p.has_media).length,
|
|
engagement_total: totalLikes,
|
|
avg_engagement: Math.round(totalLikes / posts.length),
|
|
post_types: [...new Set(posts.map(p => p.post_type))],
|
|
authors: [...new Set(posts.map(p => p.author).filter(Boolean))].length,
|
|
estimated_tokens: Math.ceil(totalTextLength / 4)
|
|
};
|
|
}
|
|
|
|
// Helper methods for extraction
|
|
extractResultTitle(element) {
|
|
const titleSelectors = ['h1, h2, h3, .title, .headline, [data-testid*="title"]'];
|
|
for (const selector of titleSelectors) {
|
|
const title = element.querySelector(selector)?.textContent?.trim();
|
|
if (title) return title.substring(0, 100);
|
|
}
|
|
return element.textContent?.trim()?.substring(0, 50) || 'No title';
|
|
}
|
|
|
|
extractResultSummary(element) {
|
|
const summarySelectors = ['.summary, .description, .snippet, .excerpt'];
|
|
for (const selector of summarySelectors) {
|
|
const summary = element.querySelector(selector)?.textContent?.trim();
|
|
if (summary) return summary.substring(0, 200);
|
|
}
|
|
return element.textContent?.trim()?.substring(0, 150) || '';
|
|
}
|
|
|
|
extractResultLink(element) {
|
|
const link = element.querySelector('a[href]')?.href ||
|
|
element.closest('a[href]')?.href ||
|
|
element.getAttribute('href');
|
|
return link || null;
|
|
}
|
|
|
|
detectResultType(element) {
|
|
if (element.textContent?.toLowerCase().includes('sponsored') ||
|
|
element.querySelector('.ad, .sponsored')) return 'sponsored';
|
|
if (element.querySelector('img, video')) return 'media';
|
|
if (element.querySelector('.price, .cost')) return 'product';
|
|
return 'organic';
|
|
}
|
|
|
|
scoreSearchResult(element) {
|
|
let score = 0.5;
|
|
if (element.querySelector('h1, h2, h3')) score += 0.2;
|
|
if (element.querySelector('img')) score += 0.1;
|
|
if (element.textContent?.length > 100) score += 0.1;
|
|
if (element.querySelector('a[href]')) score += 0.1;
|
|
return Math.min(score, 1.0);
|
|
}
|
|
|
|
extractPostText(element) {
|
|
const textSelectors = [
|
|
'[data-testid="tweetText"], .tweet-text',
|
|
'.post-content, .entry-content',
|
|
'.status-content, .message-content'
|
|
];
|
|
|
|
for (const selector of textSelectors) {
|
|
const text = element.querySelector(selector)?.textContent?.trim();
|
|
if (text) return text.substring(0, 280);
|
|
}
|
|
|
|
return element.textContent?.trim()?.substring(0, 280) || '';
|
|
}
|
|
|
|
extractPostAuthor(element) {
|
|
const authorSelectors = [
|
|
'[data-testid="User-Name"], .username',
|
|
'.author, .user-name, .handle'
|
|
];
|
|
|
|
for (const selector of authorSelectors) {
|
|
const author = element.querySelector(selector)?.textContent?.trim();
|
|
if (author) return author.substring(0, 50);
|
|
}
|
|
return 'Unknown';
|
|
}
|
|
|
|
extractPostTimestamp(element) {
|
|
const timeSelectors = ['time, .timestamp, .date, [data-testid*="time"]'];
|
|
for (const selector of timeSelectors) {
|
|
const time = element.querySelector(selector);
|
|
if (time) {
|
|
return time.getAttribute('datetime') || time.textContent?.trim() || null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
extractPostMetrics(element) {
|
|
const metrics = {};
|
|
const likeSelectors = ['[data-testid*="like"], .like-count, .heart-count'];
|
|
const replySelectors = ['[data-testid*="reply"], .reply-count, .comment-count'];
|
|
const shareSelectors = ['[data-testid*="retweet"], .share-count, .repost-count'];
|
|
|
|
for (const selector of likeSelectors) {
|
|
const likes = element.querySelector(selector)?.textContent?.match(/\d+/)?.[0];
|
|
if (likes) metrics.likes = parseInt(likes);
|
|
}
|
|
|
|
for (const selector of replySelectors) {
|
|
const replies = element.querySelector(selector)?.textContent?.match(/\d+/)?.[0];
|
|
if (replies) metrics.replies = parseInt(replies);
|
|
}
|
|
|
|
for (const selector of shareSelectors) {
|
|
const shares = element.querySelector(selector)?.textContent?.match(/\d+/)?.[0];
|
|
if (shares) metrics.shares = parseInt(shares);
|
|
}
|
|
|
|
return metrics;
|
|
}
|
|
|
|
hasPostMedia(element) {
|
|
return element.querySelector('img, video, [data-testid*="media"]') !== null;
|
|
}
|
|
|
|
detectPostType(element) {
|
|
if (element.querySelector('[data-testid*="retweet"]')) return 'repost';
|
|
if (element.querySelector('[data-testid*="reply"]')) return 'reply';
|
|
if (element.hasAttribute('data-promoted')) return 'promoted';
|
|
return 'original';
|
|
}
|
|
|
|
getTopDomains(domains, limit = 5) {
|
|
const domainCounts = {};
|
|
domains.forEach(domain => {
|
|
domainCounts[domain] = (domainCounts[domain] || 0) + 1;
|
|
});
|
|
|
|
return Object.entries(domainCounts)
|
|
.sort(([,a], [,b]) => b - a)
|
|
.slice(0, limit)
|
|
.map(([domain, count]) => ({ domain, count }));
|
|
}
|
|
|
|
calculateQualityScore(results) {
|
|
const avgScore = results.reduce((sum, r) => sum + (r.score || 0), 0) / results.length;
|
|
const hasLinks = results.filter(r => r.link).length / results.length;
|
|
const hasContent = results.filter(r => r.summary?.length > 50).length / results.length;
|
|
|
|
return Math.round((avgScore * 0.4 + hasLinks * 0.3 + hasContent * 0.3) * 100);
|
|
}
|
|
|
|
getExtractionMethod(content_type) {
|
|
const hostname = window.location.hostname;
|
|
if (hostname.includes('twitter') || hostname.includes('x.com')) return 'twitter_patterns';
|
|
if (hostname.includes('github')) return 'github_patterns';
|
|
if (hostname.includes('google')) return 'google_patterns';
|
|
return `semantic_${content_type}`;
|
|
}
|
|
|
|
estimateContentTokens(content) {
|
|
if (Array.isArray(content)) {
|
|
return content.reduce((sum, item) => {
|
|
return sum + Math.ceil(JSON.stringify(item).length / 4);
|
|
}, 0);
|
|
} else {
|
|
return Math.ceil(JSON.stringify(content).length / 4);
|
|
}
|
|
}
|
|
|
|
async clickElement({ element_id, click_type = 'left', wait_after = 500 }) {
|
|
const element = this.getElementById(element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${element_id}`);
|
|
}
|
|
|
|
// Scroll element into view
|
|
element.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
|
|
// Click the element
|
|
if (click_type === 'right') {
|
|
element.dispatchEvent(new MouseEvent('contextmenu', { bubbles: true }));
|
|
} else {
|
|
element.click();
|
|
}
|
|
|
|
await new Promise(resolve => setTimeout(resolve, wait_after));
|
|
|
|
return {
|
|
success: true,
|
|
element_id,
|
|
click_type,
|
|
element_name: this.getElementName(element)
|
|
};
|
|
}
|
|
|
|
async fillElement({ element_id, value, clear_first = true, force_focus = true }) {
|
|
const element = this.getElementById(element_id);
|
|
if (!element) {
|
|
throw new Error(`Element not found: ${element_id}`);
|
|
}
|
|
|
|
// Enhanced focus sequence for modern web apps
|
|
if (force_focus) {
|
|
await this.ensureProperFocus(element);
|
|
} else {
|
|
element.focus();
|
|
}
|
|
|
|
// Clear existing content if requested
|
|
if (clear_first) {
|
|
await this.clearElementContent(element);
|
|
}
|
|
|
|
// Fill the value with proper event sequence
|
|
await this.fillWithEvents(element, value);
|
|
|
|
// Validate the fill was successful
|
|
const actualValue = this.getElementValue(element);
|
|
const success = actualValue.includes(value);
|
|
|
|
return {
|
|
success,
|
|
element_id,
|
|
value,
|
|
actual_value: actualValue,
|
|
element_name: this.getElementName(element),
|
|
focus_applied: force_focus
|
|
};
|
|
}
|
|
|
|
async ensureProperFocus(element) {
|
|
// Scroll element into view first
|
|
element.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
|
await new Promise(resolve => setTimeout(resolve, 200));
|
|
|
|
// Simulate proper mouse interaction sequence
|
|
const rect = element.getBoundingClientRect();
|
|
const centerX = rect.left + rect.width / 2;
|
|
const centerY = rect.top + rect.height / 2;
|
|
|
|
// Fire mouse events in sequence
|
|
element.dispatchEvent(new MouseEvent('mousedown', {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY
|
|
}));
|
|
|
|
element.dispatchEvent(new MouseEvent('mouseup', {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY
|
|
}));
|
|
|
|
element.dispatchEvent(new MouseEvent('click', {
|
|
bubbles: true,
|
|
clientX: centerX,
|
|
clientY: centerY
|
|
}));
|
|
|
|
// Focus and fire focus events
|
|
element.focus();
|
|
element.dispatchEvent(new FocusEvent('focusin', { bubbles: true }));
|
|
element.dispatchEvent(new FocusEvent('focus', { bubbles: true }));
|
|
|
|
// Wait for React/framework to update
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
async clearElementContent(element) {
|
|
if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
|
|
element.value = '';
|
|
element.dispatchEvent(new Event('input', { bubbles: true }));
|
|
} else if (element.contentEditable === 'true') {
|
|
// For contenteditable, simulate selecting all and deleting
|
|
element.focus();
|
|
document.execCommand('selectAll');
|
|
document.execCommand('delete');
|
|
element.dispatchEvent(new Event('input', { bubbles: true }));
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 50));
|
|
}
|
|
|
|
async fillWithEvents(element, value) {
|
|
if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
|
|
// Set value and fire comprehensive events
|
|
element.value = value;
|
|
element.dispatchEvent(new Event('beforeinput', { bubbles: true }));
|
|
element.dispatchEvent(new Event('input', { bubbles: true }));
|
|
element.dispatchEvent(new Event('change', { bubbles: true }));
|
|
|
|
// Fire keyboard events to simulate typing completion
|
|
element.dispatchEvent(new KeyboardEvent('keydown', { bubbles: true, key: 'End' }));
|
|
element.dispatchEvent(new KeyboardEvent('keyup', { bubbles: true, key: 'End' }));
|
|
|
|
} else if (element.contentEditable === 'true') {
|
|
// For contenteditable elements (like Twitter)
|
|
element.textContent = value;
|
|
element.dispatchEvent(new Event('beforeinput', { bubbles: true }));
|
|
element.dispatchEvent(new Event('input', { bubbles: true }));
|
|
|
|
// Trigger composition events for better compatibility
|
|
element.dispatchEvent(new CompositionEvent('compositionend', {
|
|
bubbles: true,
|
|
data: value
|
|
}));
|
|
|
|
// Fire selection change to notify frameworks
|
|
document.dispatchEvent(new Event('selectionchange'));
|
|
}
|
|
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
getElementValue(element) {
|
|
if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
|
|
return element.value;
|
|
} else if (element.contentEditable === 'true') {
|
|
return element.textContent || element.innerText || '';
|
|
}
|
|
return '';
|
|
}
|
|
|
|
// Element state detection methods
|
|
getElementState(element) {
|
|
const state = {
|
|
disabled: this.isElementDisabled(element),
|
|
visible: this.isLikelyVisible(element),
|
|
clickable: this.isElementClickable(element),
|
|
focusable: this.isElementFocusable(element),
|
|
hasText: this.hasText(element),
|
|
isEmpty: this.isEmpty(element)
|
|
};
|
|
|
|
// Overall interaction readiness
|
|
state.interaction_ready = state.visible && !state.disabled && (state.clickable || state.focusable);
|
|
|
|
return state;
|
|
}
|
|
|
|
isElementDisabled(element) {
|
|
// Check disabled attribute
|
|
if (element.disabled === true) return true;
|
|
if (element.getAttribute('disabled') !== null) return true;
|
|
|
|
// Check aria-disabled
|
|
if (element.getAttribute('aria-disabled') === 'true') return true;
|
|
|
|
// Check common disabled classes
|
|
const disabledClasses = ['disabled', 'btn-disabled', 'button-disabled', 'inactive'];
|
|
const classList = Array.from(element.classList);
|
|
if (disabledClasses.some(cls => classList.includes(cls))) return true;
|
|
|
|
// Check if parent form/fieldset is disabled
|
|
const parentFieldset = element.closest('fieldset[disabled]');
|
|
if (parentFieldset) return true;
|
|
|
|
// Check computed styles for pointer-events: none
|
|
const computedStyle = getComputedStyle(element);
|
|
if (computedStyle.pointerEvents === 'none') return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
isElementClickable(element) {
|
|
const clickableTags = ['BUTTON', 'A', 'INPUT'];
|
|
const clickableTypes = ['button', 'submit', 'reset'];
|
|
const clickableRoles = ['button', 'link', 'menuitem', 'tab'];
|
|
|
|
// Check tag and type
|
|
if (clickableTags.includes(element.tagName)) return true;
|
|
if (element.type && clickableTypes.includes(element.type)) return true;
|
|
|
|
// Check role
|
|
const role = element.getAttribute('role');
|
|
if (role && clickableRoles.includes(role)) return true;
|
|
|
|
// Check for click handlers
|
|
if (element.onclick || element.getAttribute('onclick')) return true;
|
|
|
|
// Check for common clickable classes
|
|
const clickableClasses = ['btn', 'button', 'clickable', 'link'];
|
|
const classList = Array.from(element.classList);
|
|
if (clickableClasses.some(cls => classList.includes(cls))) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
isElementFocusable(element) {
|
|
const focusableTags = ['INPUT', 'TEXTAREA', 'SELECT', 'BUTTON', 'A'];
|
|
|
|
// Check if element is naturally focusable
|
|
if (focusableTags.includes(element.tagName)) return true;
|
|
|
|
// Check tabindex
|
|
const tabindex = element.getAttribute('tabindex');
|
|
if (tabindex && tabindex !== '-1') return true;
|
|
|
|
// Check contenteditable
|
|
if (element.contentEditable === 'true') return true;
|
|
|
|
// Check role
|
|
const focusableRoles = ['textbox', 'searchbox', 'button', 'link'];
|
|
const role = element.getAttribute('role');
|
|
if (role && focusableRoles.includes(role)) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
hasText(element) {
|
|
const text = element.textContent || element.value || element.getAttribute('aria-label') || '';
|
|
return text.trim().length > 0;
|
|
}
|
|
|
|
isEmpty(element) {
|
|
if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA') {
|
|
return !element.value || element.value.trim().length === 0;
|
|
}
|
|
if (element.contentEditable === 'true') {
|
|
return !element.textContent || element.textContent.trim().length === 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async waitForCondition({ condition_type, selector, text, timeout = 5000 }) {
|
|
const startTime = Date.now();
|
|
|
|
const conditions = {
|
|
'element_visible': () => {
|
|
const el = document.querySelector(selector);
|
|
return el && el.offsetParent !== null;
|
|
},
|
|
'text_present': () => document.body.textContent.includes(text)
|
|
};
|
|
|
|
const checkCondition = conditions[condition_type];
|
|
if (!checkCondition) {
|
|
throw new Error(`Unknown condition type: ${condition_type}`);
|
|
}
|
|
|
|
while (Date.now() - startTime < timeout) {
|
|
if (checkCondition()) {
|
|
return {
|
|
condition_met: true,
|
|
wait_time: Date.now() - startTime
|
|
};
|
|
}
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
throw new Error(`Timeout waiting for condition: ${condition_type}`);
|
|
}
|
|
|
|
// Utility methods
|
|
registerElement(element) {
|
|
const id = `element_${++this.idCounter}`;
|
|
this.elementRegistry.set(id, element);
|
|
return id;
|
|
}
|
|
|
|
getElementById(id) {
|
|
// Check quick registry first (for q1, q2, etc.)
|
|
if (id.startsWith('q')) {
|
|
return this.quickRegistry.get(id);
|
|
}
|
|
// Then check main registry (for element_1, element_2, etc.)
|
|
return this.elementRegistry.get(id);
|
|
}
|
|
|
|
getElementName(element) {
|
|
return element.getAttribute('aria-label') ||
|
|
element.getAttribute('title') ||
|
|
element.textContent?.trim()?.substring(0, 50) ||
|
|
element.placeholder ||
|
|
element.tagName.toLowerCase();
|
|
}
|
|
|
|
isVisible(element) {
|
|
return element.offsetParent !== null &&
|
|
getComputedStyle(element).visibility !== 'hidden' &&
|
|
getComputedStyle(element).opacity !== '0';
|
|
}
|
|
|
|
generateSelector(element) {
|
|
if (element.id) return `#${element.id}`;
|
|
if (element.getAttribute('data-testid')) return `[data-testid="${element.getAttribute('data-testid')}"]`;
|
|
|
|
let selector = element.tagName.toLowerCase();
|
|
if (element.className) {
|
|
selector += `.${element.className.split(' ').join('.')}`;
|
|
}
|
|
return selector;
|
|
}
|
|
|
|
inferElementType(element, intentHint) {
|
|
const tagName = element.tagName.toLowerCase();
|
|
const role = element.getAttribute('role');
|
|
const type = element.getAttribute('type');
|
|
|
|
if (tagName === 'input' && type === 'search') return 'search_input';
|
|
if (tagName === 'input') return 'input';
|
|
if (tagName === 'textarea') return 'textarea';
|
|
if (tagName === 'button' || role === 'button') return 'button';
|
|
if (tagName === 'a') return 'link';
|
|
|
|
return 'element';
|
|
}
|
|
|
|
calculateConfidence(element, intentHint) {
|
|
let confidence = 0.5;
|
|
|
|
const text = this.getElementName(element).toLowerCase();
|
|
const hint = intentHint.toLowerCase();
|
|
|
|
if (text.includes(hint)) confidence += 0.3;
|
|
if (element.getAttribute('data-testid')) confidence += 0.2;
|
|
if (element.getAttribute('aria-label')) confidence += 0.1;
|
|
|
|
return Math.min(confidence, 1.0);
|
|
}
|
|
|
|
formatAnalysisResult(result, method, startTime) {
|
|
return {
|
|
...result,
|
|
method,
|
|
execution_time: Math.round(performance.now() - startTime),
|
|
analyzed_at: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// Two-phase utility methods
|
|
compressElement(element, isQuick = false) {
|
|
const actualElement = element.element || element;
|
|
const state = this.getElementState(actualElement);
|
|
|
|
if (isQuick) {
|
|
// Quick phase - minimal data with state
|
|
const quickId = `q${++this.quickIdCounter}`;
|
|
this.quickRegistry.set(quickId, actualElement);
|
|
|
|
return {
|
|
id: quickId,
|
|
type: element.type || 'element',
|
|
name: element.name?.substring(0, 20) || 'unnamed',
|
|
conf: Math.round((element.confidence || 0.5) * 100),
|
|
selector: element.selector || 'unknown',
|
|
state: state.disabled ? 'disabled' : 'enabled',
|
|
clickable: state.clickable,
|
|
ready: state.interaction_ready
|
|
};
|
|
} else {
|
|
// Detailed phase - compact fingerprint with full state
|
|
return {
|
|
id: element.id,
|
|
fp: this.generateFingerprint(actualElement),
|
|
name: element.name?.substring(0, 30) || 'unnamed',
|
|
conf: Math.round((element.confidence || 0.5) * 100),
|
|
meta: {
|
|
...this.getElementMeta(actualElement),
|
|
state: state
|
|
}
|
|
};
|
|
}
|
|
}
|
|
|
|
generateFingerprint(element) {
|
|
const tag = element.tagName.toLowerCase();
|
|
const primaryClass = this.getPrimaryClass(element);
|
|
const context = this.getContext(element);
|
|
const position = this.getRelativePosition(element);
|
|
|
|
return `${tag}${primaryClass ? '.' + primaryClass : ''}@${context}.${position}`;
|
|
}
|
|
|
|
getPrimaryClass(element) {
|
|
const importantClasses = ['btn', 'button', 'link', 'input', 'search', 'submit', 'primary', 'secondary'];
|
|
const classList = Array.from(element.classList);
|
|
return classList.find(cls => importantClasses.includes(cls)) || classList[0];
|
|
}
|
|
|
|
getContext(element) {
|
|
const parent = element.closest('nav, main, header, footer, form, section, article') || element.parentElement;
|
|
if (!parent) return 'body';
|
|
return parent.tagName.toLowerCase();
|
|
}
|
|
|
|
getRelativePosition(element) {
|
|
const siblings = Array.from(element.parentElement?.children || []);
|
|
const sameTypeElements = siblings.filter(el => el.tagName === element.tagName);
|
|
return sameTypeElements.indexOf(element) + 1;
|
|
}
|
|
|
|
getElementMeta(element) {
|
|
const rect = element.getBoundingClientRect();
|
|
return {
|
|
rect: [Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height)],
|
|
visible: this.isLikelyVisible(element),
|
|
form_context: element.closest('form') ? 'form' : null
|
|
};
|
|
}
|
|
|
|
detectPageType() {
|
|
const hostname = window.location.hostname;
|
|
const title = document.title.toLowerCase();
|
|
const hasSearch = document.querySelector('[type="search"], [role="searchbox"]');
|
|
const hasLogin = document.querySelector('[type="password"], [name*="login" i]');
|
|
const hasPost = document.querySelector('[contenteditable="true"], textarea[placeholder*="post" i]');
|
|
|
|
if (hostname.includes('twitter') || hostname.includes('x.com')) return 'social_media';
|
|
if (hostname.includes('github')) return 'code_repository';
|
|
if (hostname.includes('google')) return 'search_engine';
|
|
if (hasPost) return 'content_creation';
|
|
if (hasLogin) return 'authentication';
|
|
if (hasSearch) return 'search_interface';
|
|
if (title.includes('shop') || title.includes('store')) return 'ecommerce';
|
|
|
|
return 'general_website';
|
|
}
|
|
|
|
countViewportElements() {
|
|
const elements = document.querySelectorAll('button, input, select, textarea, a[href]');
|
|
const viewportElements = Array.from(elements).filter(el => this.isLikelyVisible(el));
|
|
|
|
return {
|
|
buttons: viewportElements.filter(el => el.tagName === 'BUTTON' || el.getAttribute('role') === 'button').length,
|
|
inputs: viewportElements.filter(el => el.tagName === 'INPUT').length,
|
|
links: viewportElements.filter(el => el.tagName === 'A').length,
|
|
textareas: viewportElements.filter(el => el.tagName === 'TEXTAREA').length,
|
|
selects: viewportElements.filter(el => el.tagName === 'SELECT').length
|
|
};
|
|
}
|
|
|
|
async quickViewportScan(intent_hint, maxResults = 3) {
|
|
const candidates = document.querySelectorAll('button, input, a[href], [role="button"], textarea');
|
|
const visibleElements = Array.from(candidates)
|
|
.filter(el => this.isLikelyVisible(el))
|
|
.slice(0, 10); // Limit scan to first 10 visible elements
|
|
|
|
const scoredElements = visibleElements.map(element => {
|
|
const confidence = this.calculateConfidence(element, intent_hint);
|
|
return {
|
|
element,
|
|
type: this.inferElementType(element, intent_hint),
|
|
name: this.getElementName(element),
|
|
confidence
|
|
};
|
|
});
|
|
|
|
return scoredElements
|
|
.filter(el => el.confidence > 0.3)
|
|
.sort((a, b) => b.confidence - a.confidence)
|
|
.slice(0, maxResults)
|
|
.map(el => this.compressElement(el, true));
|
|
}
|
|
|
|
scoreIntentMatch(intent_hint, quickMatches) {
|
|
if (quickMatches.length === 0) return 'none';
|
|
const avgConfidence = quickMatches.reduce((sum, match) => sum + match.conf, 0) / quickMatches.length;
|
|
|
|
if (avgConfidence >= 80) return 'high';
|
|
if (avgConfidence >= 60) return 'medium';
|
|
if (avgConfidence >= 40) return 'low';
|
|
return 'none';
|
|
}
|
|
|
|
suggestPhase2Areas(quickMatches, intent_hint) {
|
|
const suggestions = [];
|
|
const elementTypes = [...new Set(quickMatches.map(m => m.type))];
|
|
|
|
if (elementTypes.includes('button')) suggestions.push('buttons');
|
|
if (elementTypes.includes('input') || elementTypes.includes('textarea')) suggestions.push('forms');
|
|
if (elementTypes.includes('link')) suggestions.push('navigation');
|
|
|
|
// Intent-based suggestions
|
|
if (intent_hint.toLowerCase().includes('search') && !suggestions.includes('forms')) {
|
|
suggestions.push('search_elements');
|
|
}
|
|
|
|
return suggestions.slice(0, 3);
|
|
}
|
|
|
|
estimatePhase2Tokens(quickMatches) {
|
|
// Estimate tokens needed for detailed analysis
|
|
const baseTokens = 50; // Base overhead
|
|
const tokensPerElement = 15; // Detailed element info
|
|
const contextTokens = 20; // Page context
|
|
|
|
return baseTokens + (quickMatches.length * tokensPerElement) + contextTokens;
|
|
}
|
|
|
|
async expandQuickMatches(element_ids) {
|
|
const elements = [];
|
|
for (const id of element_ids) {
|
|
const element = this.quickRegistry.get(id);
|
|
if (element) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, ''),
|
|
name: this.getElementName(element),
|
|
confidence: 0.8, // Default confidence for expanded elements
|
|
element: element
|
|
});
|
|
}
|
|
}
|
|
return elements;
|
|
}
|
|
|
|
async analyzeFocusAreas(focus_areas, intent_hint) {
|
|
const elements = [];
|
|
const areaSelectors = {
|
|
'buttons': 'button, [role="button"], input[type="submit"]',
|
|
'forms': 'input, textarea, select, [contenteditable="true"]',
|
|
'navigation': 'nav a, .nav-item, [role="navigation"] a',
|
|
'search_elements': '[type="search"], [role="searchbox"], [placeholder*="search" i]'
|
|
};
|
|
|
|
for (const area of focus_areas) {
|
|
const selector = areaSelectors[area];
|
|
if (selector) {
|
|
const areaElements = document.querySelectorAll(selector);
|
|
for (const element of Array.from(areaElements).slice(0, 5)) {
|
|
if (this.isLikelyVisible(element)) {
|
|
const elementId = this.registerElement(element);
|
|
elements.push({
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intent_hint),
|
|
element: element
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return elements;
|
|
}
|
|
|
|
async fullEnhancedAnalysis(intent_hint, max_results) {
|
|
// Enhanced version of semantic analysis with better filtering
|
|
const relevantElements = document.querySelectorAll(`
|
|
button, input, select, textarea, a[href],
|
|
[role="button"], [role="textbox"], [role="searchbox"],
|
|
[aria-label], [data-testid], [contenteditable="true"]
|
|
`);
|
|
|
|
const elements = Array.from(relevantElements)
|
|
.filter(el => this.isLikelyVisible(el))
|
|
.slice(0, 30) // Analyze more elements than before
|
|
.map(element => {
|
|
const elementId = this.registerElement(element);
|
|
return {
|
|
id: elementId,
|
|
type: this.inferElementType(element, intent_hint),
|
|
selector: this.generateSelector(element),
|
|
name: this.getElementName(element),
|
|
confidence: this.calculateConfidence(element, intent_hint),
|
|
element: element
|
|
};
|
|
})
|
|
.filter(el => el.confidence > 0.2) // Lower threshold for detailed analysis
|
|
.sort((a, b) => b.confidence - a.confidence);
|
|
|
|
return elements.slice(0, max_results);
|
|
}
|
|
|
|
deduplicateElements(elements) {
|
|
const seen = new Set();
|
|
return elements.filter(element => {
|
|
const key = element.name + element.type;
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
async enhanceElementMetadata(elements) {
|
|
return elements.map(element => ({
|
|
...element,
|
|
meta: this.getElementMeta(element.element)
|
|
}));
|
|
}
|
|
|
|
isLikelyVisible(element) {
|
|
const rect = element.getBoundingClientRect();
|
|
const style = getComputedStyle(element);
|
|
|
|
return rect.top < window.innerHeight &&
|
|
rect.bottom > 0 &&
|
|
rect.left < window.innerWidth &&
|
|
rect.right > 0 &&
|
|
style.visibility !== 'hidden' &&
|
|
style.opacity !== '0' &&
|
|
style.display !== 'none';
|
|
}
|
|
|
|
estimateTokenUsage(result) {
|
|
// Estimate token count based on result size
|
|
const jsonString = JSON.stringify(result);
|
|
return Math.ceil(jsonString.length / 4); // Rough estimate: 4 chars per token
|
|
}
|
|
}
|
|
|
|
// Initialize the automation system
|
|
const browserAutomation = new BrowserAutomation(); |