fix(humanize): port #303 iframe pointer-events fix to Python

Mirror the JS fix from #303 in the sync and async Python actionability
checks: compute and apply the iframe coordinate offset before
elementFromPoint, and fail open when the check itself cannot run. Add
fail-open regression tests for both Python and JS.
This commit is contained in:
CloakHQ
2026-05-25 01:17:05 +02:00
parent 12d02c3547
commit 7fc577e5c6
4 changed files with 125 additions and 18 deletions
+23 -11
View File
@@ -196,8 +196,14 @@ def ensure_stable(
# Pointer-events check (post-scroll, at actual click coordinates)
# ---------------------------------------------------------------------------
_POINTER_EVENTS_LOCATOR_JS = """(expected, coords) => {
const target = document.elementFromPoint(coords.x, coords.y);
# data.box is page-space (from bounding_box); rect is frame-local. Their delta
# is the iframe offset, needed to map page-space click coords into the frame's
# own viewport before elementFromPoint. For main-frame elements the offset is 0.
_POINTER_EVENTS_LOCATOR_JS = """(expected, data) => {
const rect = expected.getBoundingClientRect();
const frameOffsetX = data.box ? data.box.x - rect.x : 0;
const frameOffsetY = data.box ? data.box.y - rect.y : 0;
const target = document.elementFromPoint(data.x - frameOffsetX, data.y - frameOffsetY);
if (!target) return { hit: false, reason: 'no_element_at_point', covering: 'none' };
let node = target;
while (node) { if (node === expected) return { hit: true }; node = node.parentNode; }
@@ -205,8 +211,11 @@ _POINTER_EVENTS_LOCATOR_JS = """(expected, coords) => {
return { hit: false, reason: 'covered', covering: target.tagName || 'unknown' };
}"""
_POINTER_EVENTS_HANDLE_JS = """(expected, coords) => {
const target = document.elementFromPoint(coords.x, coords.y);
_POINTER_EVENTS_HANDLE_JS = """(expected, data) => {
const rect = expected.getBoundingClientRect();
const frameOffsetX = data.box ? data.box.x - rect.x : 0;
const frameOffsetY = data.box ? data.box.y - rect.y : 0;
const target = document.elementFromPoint(data.x - frameOffsetX, data.y - frameOffsetY);
if (!target) return { hit: false, reason: 'no_element_at_point', covering: 'none' };
let node = target;
while (node) { if (node === expected) return { hit: true }; node = node.parentNode; }
@@ -230,17 +239,19 @@ def check_pointer_events(
"""
deadline = time.monotonic() + timeout / 1000.0
attempt = 0
coords = {"x": x, "y": y}
while True:
try:
loc = page.locator(selector).first
result = loc.evaluate(_POINTER_EVENTS_LOCATOR_JS, coords)
box = loc.bounding_box(timeout=max(1, min((deadline - time.monotonic()) * 1000, 1000)))
result = loc.evaluate(_POINTER_EVENTS_LOCATOR_JS, {"x": x, "y": y, "box": box})
except Exception as exc:
logger.debug("pointer_events check failed for %r: %s", selector, exc)
result = None
if result and result.get("hit", False):
# Proceed if the check confirms a hit, or if it could not be determined
# (None) — failing closed would block legitimate clicks.
if result is None or result.get("hit", False):
return
covering = (result or {}).get("covering", "unknown")
@@ -322,15 +333,16 @@ def check_pointer_events_handle(
deadline = time.monotonic() + timeout / 1000.0
attempt = 0
coords = {"x": x, "y": y}
while True:
try:
result = el.evaluate(_POINTER_EVENTS_HANDLE_JS, coords)
box = el.bounding_box()
result = el.evaluate(_POINTER_EVENTS_HANDLE_JS, {"x": x, "y": y, "box": box})
except Exception:
result = None
if result and result.get("hit", False):
# Proceed if the check confirms a hit, or if it could not be determined
# (None) — failing closed would block legitimate clicks.
if result is None or result.get("hit", False):
return
covering = (result or {}).get("covering", "unknown")
+10 -7
View File
@@ -140,17 +140,19 @@ async def async_check_pointer_events(
) -> None:
deadline = time.monotonic() + timeout / 1000.0
attempt = 0
coords = {"x": x, "y": y}
while True:
try:
loc = page.locator(selector).first
result = await loc.evaluate(_POINTER_EVENTS_LOCATOR_JS, coords)
box = await loc.bounding_box(timeout=max(1, min((deadline - time.monotonic()) * 1000, 1000)))
result = await loc.evaluate(_POINTER_EVENTS_LOCATOR_JS, {"x": x, "y": y, "box": box})
except Exception as exc:
logger.debug("pointer_events check failed for %r: %s", selector, exc)
result = None
if result and result.get("hit", False):
# Proceed if the check confirms a hit, or if it could not be determined
# (None) — failing closed would block legitimate clicks.
if result is None or result.get("hit", False):
return
covering = (result or {}).get("covering", "unknown")
@@ -227,15 +229,16 @@ async def async_check_pointer_events_handle(
deadline = time.monotonic() + timeout / 1000.0
attempt = 0
coords = {"x": x, "y": y}
while True:
try:
result = await el.evaluate(_POINTER_EVENTS_HANDLE_JS, coords)
box = await el.bounding_box()
result = await el.evaluate(_POINTER_EVENTS_HANDLE_JS, {"x": x, "y": y, "box": box})
except Exception:
result = None
if result and result.get("hit", False):
# Proceed if the check confirms a hit, or if it could not be determined
# (None) — failing closed would block legitimate clicks.
if result is None or result.get("hit", False):
return
covering = (result or {}).get("covering", "unknown")
+40
View File
@@ -1559,3 +1559,43 @@ describe("frame.click timeout budget (#307)", () => {
expect(elapsed).toBeLessThan(TIMEOUT_MS * 1.8);
});
});
describe("pointer-events check fail-open", () => {
// When the check itself cannot run (evaluate / boundingBox throws -> result
// null), proceed with the click instead of blocking it until the timeout.
it("checkPointerEventsHandle returns promptly when evaluate throws", async () => {
const { checkPointerEventsHandle } = await import("../src/human/actionability.js");
const el = {
boundingBox: vi.fn().mockRejectedValue(new Error("stale handle")),
evaluate: vi.fn().mockRejectedValue(new Error("execution context destroyed")),
};
const start = Date.now();
await checkPointerEventsHandle(el as any, 100, 100, 2000); // must not throw
expect(Date.now() - start).toBeLessThan(500);
});
it("checkPointerEvents returns promptly when evaluate throws", async () => {
const { checkPointerEvents } = await import("../src/human/actionability.js");
const loc = {
first: () => loc,
boundingBox: vi.fn().mockRejectedValue(new Error("no element")),
evaluate: vi.fn().mockRejectedValue(new Error("no element")),
};
const page = { locator: vi.fn().mockReturnValue(loc) };
const start = Date.now();
await checkPointerEvents(page as any, "#x", 100, 100, null, 2000); // must not throw
expect(Date.now() - start).toBeLessThan(500);
});
it("checkPointerEventsHandle still throws when genuinely covered", async () => {
const { checkPointerEventsHandle, ElementNotReceivingEventsError } =
await import("../src/human/actionability.js");
const el = {
boundingBox: vi.fn().mockResolvedValue({ x: 0, y: 0, width: 10, height: 10 }),
evaluate: vi.fn().mockResolvedValue({ hit: false, covering: "DIV" }),
};
await expect(checkPointerEventsHandle(el as any, 5, 5, 200)).rejects.toBeInstanceOf(
ElementNotReceivingEventsError,
);
});
});
+52
View File
@@ -1888,6 +1888,58 @@ class TestTimeoutBudget307:
)
class TestPointerEventsFailOpen:
"""The pointer-events check must fail open: when it cannot run (evaluate /
bounding_box throws -> result None), proceed with the click instead of
blocking it until the timeout expires."""
def test_handle_failopen_returns_on_evaluate_error(self):
from cloakbrowser.human.actionability import check_pointer_events_handle
el = MagicMock()
el.bounding_box = MagicMock(side_effect=Exception("stale handle"))
el.evaluate = MagicMock(side_effect=Exception("execution context destroyed"))
start = time.monotonic()
check_pointer_events_handle(MagicMock(), el, 100, 100, timeout=2000) # must not raise
elapsed_ms = (time.monotonic() - start) * 1000
assert elapsed_ms < 500, f"fail-open should return promptly, took {elapsed_ms:.0f}ms"
def test_locator_failopen_returns_on_evaluate_error(self):
from cloakbrowser.human.actionability import check_pointer_events
page = MagicMock()
loc = MagicMock()
loc.first = loc
loc.bounding_box = MagicMock(side_effect=Exception("no element"))
loc.evaluate = MagicMock(side_effect=Exception("no element"))
page.locator = MagicMock(return_value=loc)
start = time.monotonic()
check_pointer_events(page, "#x", 100, 100, timeout=2000) # must not raise
elapsed_ms = (time.monotonic() - start) * 1000
assert elapsed_ms < 500, f"fail-open should return promptly, took {elapsed_ms:.0f}ms"
def test_handle_still_raises_when_covered(self):
"""A genuine 'covered' result (not None) must still raise — fail-open
only applies when the check could not be determined."""
from cloakbrowser.human.actionability import (
check_pointer_events_handle, ElementNotReceivingEventsError,
)
el = MagicMock()
el.bounding_box = MagicMock(return_value={"x": 0, "y": 0, "width": 10, "height": 10})
el.evaluate = MagicMock(return_value={"hit": False, "covering": "DIV"})
with pytest.raises(ElementNotReceivingEventsError):
check_pointer_events_handle(MagicMock(), el, 5, 5, timeout=200)
def test_async_handle_failopen_returns_on_evaluate_error(self):
from cloakbrowser.human.actionability_async import async_check_pointer_events_handle
from unittest.mock import AsyncMock
el = MagicMock()
el.bounding_box = AsyncMock(side_effect=Exception("stale handle"))
el.evaluate = AsyncMock(side_effect=Exception("execution context destroyed"))
start = time.monotonic()
asyncio.run(async_check_pointer_events_handle(MagicMock(), el, 100, 100, timeout=2000))
elapsed_ms = (time.monotonic() - start) * 1000
assert elapsed_ms < 500, f"fail-open should return promptly, took {elapsed_ms:.0f}ms"
# =========================================================================
# Direct runner (backwards compat)
# =========================================================================