From 114b3c826b7b01ab176f3dfcf22862517e483489 Mon Sep 17 00:00:00 2001 From: manaskarra <106862611+manaskarra@users.noreply.github.com> Date: Sun, 10 May 2026 20:13:29 +0400 Subject: [PATCH] fix(js): preserve iframe scope in humanized frame actions (#201) fix(js): preserve iframe scope in humanized frame actions Frame actions (click, type, fill, etc.) now resolve selectors through frame.locator() instead of delegating to page.* methods, fixing iframe-scoped interactions. Closes #184 Co-authored-by: manaskarra --- js/src/human/index.ts | 129 ++++++++++++++++++++++++++++++++------ js/tests/humanize.test.ts | 77 ++++++++++++++++++++++- 2 files changed, 184 insertions(+), 22 deletions(-) diff --git a/js/src/human/index.ts b/js/src/human/index.ts index 58e725b..a15d2a7 100644 --- a/js/src/human/index.ts +++ b/js/src/human/index.ts @@ -529,16 +529,37 @@ function patchFrames( stealth: StealthEval, ): void { for (const frame of iterFrames(page)) { - patchSingleFrame(frame, page, cfg, originals, stealth); + patchSingleFrame(frame, page, cfg, cursor, raw, rawKb, originals, stealth); // Patch frame-level ElementHandle selectors ($, $$, waitForSelector) patchFrameElementHandles(frame, page, cfg, cursor, raw, rawKb, originals, stealth); } } +function firstFrameLocator(frame: Frame, selector: string): any { + const locator = frame.locator(selector) as any; + return typeof locator.first === 'function' ? locator.first() : locator; +} + +async function isFrameInputElement(frame: Frame, selector: string): Promise { + return firstFrameLocator(frame, selector).evaluate((el: Element) => { + const tag = el.tagName.toLowerCase(); + return tag === 'input' || tag === 'textarea' + || el.getAttribute('contenteditable') === 'true'; + }).catch(() => false); +} + +async function isFrameSelectorFocused(frame: Frame, selector: string): Promise { + return firstFrameLocator(frame, selector).evaluate((el: Element) => el === document.activeElement) + .catch(() => false); +} + function patchSingleFrame( frame: Frame, page: Page, cfg: HumanConfig, + cursor: CursorState, + raw: RawMouse, + rawKb: RawKeyboard, originals: any, stealth: StealthEval, ): void { @@ -546,58 +567,128 @@ function patchSingleFrame( (frame as any)._humanPatched = true; // Save originals for methods that need fallback + const origFrameClick = frame.click.bind(frame); + const origFrameDblclick = frame.dblclick.bind(frame); + const origFrameHover = frame.hover.bind(frame); + const origFrameType = frame.type.bind(frame); + const origFrameFill = frame.fill.bind(frame); + const origFrameCheck = frame.check.bind(frame); + const origFrameUncheck = frame.uncheck.bind(frame); const origFrameSelectOption = frame.selectOption.bind(frame); + const origFramePress = frame.press.bind(frame); + const origFramePressSequentially = (frame as any).pressSequentially?.bind(frame); + const origFrameTap = (frame as any).tap?.bind(frame); const origFrameDragAndDrop = frame.dragAndDrop.bind(frame); - (frame as any).click = async (selector: string, options?: any) => { - await (page as any).click(selector, options); + const moveToFrameSelector = async (selector: string, options?: any, inputBias = false) => { + const callCfg = mergeConfig(cfg, options?.human_config); + if (callCfg.idle_between_actions) { + await humanIdle(raw, rand(callCfg.idle_between_duration[0], callCfg.idle_between_duration[1]), cursor.x, cursor.y, callCfg); + } + + const locator = firstFrameLocator(frame, selector); + if (typeof locator.scrollIntoViewIfNeeded === 'function') { + await locator.scrollIntoViewIfNeeded({ timeout: options?.timeout }).catch(() => undefined); + } + const box = await locator.boundingBox({ timeout: options?.timeout ?? 30000 }).catch(() => null); + if (!box) return null; + + const isInput = inputBias || await isFrameInputElement(frame, selector); + const target = clickTarget(box, isInput, callCfg); + await humanMove(raw, cursor.x, cursor.y, target.x, target.y, callCfg); + cursor.x = target.x; + cursor.y = target.y; + return { callCfg, isInput }; }; + const frameClick = async (selector: string, options?: any) => { + const moved = await moveToFrameSelector(selector, options); + if (!moved) return origFrameClick(selector, options); + await humanClick(raw, moved.isInput, moved.callCfg); + }; + + const getFrameCdp = async () => stealth.getCdpSession().catch(() => null); + + const frameHover = async (selector: string, options?: any) => { + const moved = await moveToFrameSelector(selector, options, false); + if (!moved) return origFrameHover(selector, options); + }; + + (frame as any).click = frameClick; + (frame as any).dblclick = async (selector: string, options?: any) => { - await (page as any).dblclick(selector, options); + const moved = await moveToFrameSelector(selector, options); + if (!moved) return origFrameDblclick(selector, options); + await raw.down({ clickCount: 2 }); + await sleep(rand(30, 60)); + await raw.up({ clickCount: 2 }); }; - (frame as any).hover = async (selector: string, options?: any) => { - await (page as any).hover(selector, options); - }; + (frame as any).hover = frameHover; (frame as any).type = async (selector: string, text: string, options?: any) => { - await (page as any).type(selector, text, options); + const callCfg = mergeConfig(cfg, options?.human_config); + await sleep(randRange(callCfg.field_switch_delay)); + await frameClick(selector, options); + await sleep(rand(100, 250)); + const cdp = await getFrameCdp(); + await humanType(page, rawKb, text, callCfg, cdp).catch(() => origFrameType(selector, text, options)); }; (frame as any).fill = async (selector: string, value: string, options?: any) => { - await (page as any).fill(selector, value, options); + const callCfg = mergeConfig(cfg, options?.human_config); + await sleep(randRange(callCfg.field_switch_delay)); + await frameClick(selector, options); + await sleep(rand(100, 250)); + await originals.keyboardPress(SELECT_ALL); + await sleep(rand(30, 80)); + await originals.keyboardPress('Backspace'); + await sleep(rand(50, 150)); + const cdp = await getFrameCdp(); + await humanType(page, rawKb, value, callCfg, cdp).catch(() => origFrameFill(selector, value, options)); }; (frame as any).check = async (selector: string, options?: any) => { - await (page as any).check(selector, options); + const checked = await firstFrameLocator(frame, selector).isChecked?.().catch(() => false) ?? false; + if (!checked) await frameClick(selector, options).catch(() => origFrameCheck(selector, options)); }; (frame as any).uncheck = async (selector: string, options?: any) => { - await (page as any).uncheck(selector, options); + const checked = await firstFrameLocator(frame, selector).isChecked?.().catch(() => true) ?? true; + if (checked) await frameClick(selector, options).catch(() => origFrameUncheck(selector, options)); }; (frame as any).selectOption = async (selector: string, values: any, options?: any) => { - await (page as any).hover(selector); + await frameHover(selector, options); await sleep(rand(100, 300)); return origFrameSelectOption(selector, values, options); }; (frame as any).press = async (selector: string, key: string, options?: any) => { - await (page as any).press(selector, key, options); + if (!await isFrameSelectorFocused(frame, selector)) { + await frameClick(selector, options); + } + await sleep(rand(50, 150)); + await originals.keyboardPress(key); }; (frame as any).pressSequentially = async (selector: string, text: string, options?: any) => { - await (page as any).pressSequentially(selector, text, options); + const callCfg = mergeConfig(cfg, options?.human_config); + if (!await isFrameSelectorFocused(frame, selector)) { + await frameClick(selector, options); + } + await sleep(rand(100, 250)); + const cdp = await getFrameCdp(); + await humanType(page, rawKb, text, callCfg, cdp).catch(() => origFramePressSequentially?.(selector, text, options)); }; (frame as any).tap = async (selector: string, options?: any) => { - await (page as any).tap(selector, options); + await frameClick(selector, options).catch(() => origFrameTap?.(selector, options)); }; (frame as any).clear = async (selector: string, options?: any) => { - if (!await isSelectorFocused(stealth, page, selector)) { - await (page as any).click(selector); + if (!await isFrameSelectorFocused(frame, selector)) { + await frameClick(selector, options); } await sleep(rand(50, 150)); await originals.keyboardPress(SELECT_ALL); @@ -606,8 +697,8 @@ function patchSingleFrame( }; (frame as any).dragAndDrop = async (source: string, target: string, options?: any) => { - const srcBox = await frame.locator(source).boundingBox().catch(() => null); - const tgtBox = await frame.locator(target).boundingBox().catch(() => null); + const srcBox = await firstFrameLocator(frame, source).boundingBox({ timeout: options?.timeout ?? 30000 }).catch(() => null); + const tgtBox = await firstFrameLocator(frame, target).boundingBox({ timeout: options?.timeout ?? 30000 }).catch(() => null); if (srcBox && tgtBox) { const sx = srcBox.x + srcBox.width / 2; diff --git a/js/tests/humanize.test.ts b/js/tests/humanize.test.ts index 836ae38..6afcc3a 100644 --- a/js/tests/humanize.test.ts +++ b/js/tests/humanize.test.ts @@ -398,6 +398,69 @@ describe("patchPage frame patching", () => { expect((childFrame as any)._humanPatched).toBe(true); }); + + it("uses frame.locator for frame.click instead of page.click", async () => { + const { patchPage } = await import("../src/human/index.js"); + + const childFrame = buildMockFrame(); + const mainFrame = { + ...buildMockFrame(), + childFrames: vi.fn(() => [childFrame]), + }; + const page = buildMockPage({ mainFrameReturn: mainFrame }); + const originalPageClick = page.click; + const cfg = resolveConfig("default", { mouse_min_steps: 1, mouse_max_steps: 1 }); + const cursor = { x: 0, y: 0, initialized: true }; + patchPage(page as any, cfg, cursor as any); + + await (childFrame as any).click("button.submit", { timeout: 1234 }); + + expect(childFrame.locator).toHaveBeenCalledWith("button.submit"); + expect(originalPageClick).not.toHaveBeenCalled(); + }); + + it.each([ + ["type", async (frame: any) => frame.type("input.email", "@")], + ["fill", async (frame: any) => frame.fill("input.email", "@")], + ["pressSequentially", async (frame: any) => frame.pressSequentially("input.email", "@")], + ])("passes the page CDP session to frame.%s", async (_name, runFrameAction) => { + const { patchPage } = await import("../src/human/index.js"); + + const cdpSend = vi.fn(async () => ({})); + const childFrame = buildMockFrame(); + const mainFrame = { + ...buildMockFrame(), + childFrames: vi.fn(() => [childFrame]), + }; + const page = buildMockPage({ mainFrameReturn: mainFrame }); + page.context = vi.fn(() => ({ + pages: vi.fn(() => []), + addInitScript: vi.fn(async () => {}), + newCDPSession: vi.fn(async () => ({ send: cdpSend })), + })); + + const cfg = resolveConfig("default", { + field_switch_delay: [0, 0], + key_hold: [0, 0], + shift_down_delay: [0, 0], + shift_up_delay: [0, 0], + typing_delay: 0, + typing_delay_spread: 0, + typing_pause_chance: 0, + mistype_chance: 0, + mouse_min_steps: 1, + mouse_max_steps: 1, + idle_between_actions: false, + }); + const cursor = { x: 0, y: 0, initialized: true }; + patchPage(page as any, cfg, cursor as any); + + await runFrameAction(childFrame); + + const dispatches = cdpSend.mock.calls.filter(([method]) => method === "Input.dispatchKeyEvent"); + expect(dispatches).toHaveLength(2); + expect(page.evaluate).not.toHaveBeenCalled(); + }); }); // ========================================================================= @@ -1034,6 +1097,14 @@ describe("patchPage integrates ElementHandle patching", () => { function buildMockFrame(): any { + const locator: any = { + boundingBox: vi.fn(async () => ({ x: 0, y: 0, width: 100, height: 30 })), + scrollIntoViewIfNeeded: vi.fn(async () => {}), + evaluate: vi.fn(async () => false), + isChecked: vi.fn(async () => false), + }; + locator.first = vi.fn(() => locator); + return { click: vi.fn(async () => {}), dblclick: vi.fn(async () => {}), @@ -1044,11 +1115,11 @@ function buildMockFrame(): any { uncheck: vi.fn(async () => {}), selectOption: vi.fn(async () => {}), press: vi.fn(async () => {}), + pressSequentially: vi.fn(async () => {}), + tap: vi.fn(async () => {}), clear: vi.fn(async () => {}), dragAndDrop: vi.fn(async () => {}), - locator: vi.fn(() => ({ - boundingBox: vi.fn(async () => ({ x: 0, y: 0, width: 100, height: 30 })), - })), + locator: vi.fn(() => locator), childFrames: vi.fn(() => []), }; }