diff --git a/README.md b/README.md index c51400d..8726dc2 100644 --- a/README.md +++ b/README.md @@ -631,6 +631,7 @@ browser = await launch_async(args=["--remote-debugging-port=9242"]) |-----------|-------|----------|---------| | [browser-use](https://github.com/browser-use/browser-use) | 70K | Python | [`browser_use_example.py`](examples/integrations/browser_use_example.py) | | [Crawl4AI](https://github.com/unclecode/crawl4ai) | 58K | Python | [`crawl4ai_example.py`](examples/integrations/crawl4ai_example.py) | +| [Crawlee](https://github.com/apify/crawlee-python) | 8.6K | Python | [`crawlee_example.py`](examples/integrations/crawlee_example.py) | | [Scrapling](https://github.com/D4Vinci/Scrapling) | 21K | Python | [`scrapling_example.py`](examples/integrations/scrapling_example.py) | | [Stagehand](https://github.com/browserbase/stagehand) | 21K | TypeScript | [`stagehand.ts`](js/examples/stagehand.ts) | | [LangChain](https://github.com/langchain-ai/langchain) | 100K+ | Python | [`langchain_loader.py`](examples/integrations/langchain_loader.py) | diff --git a/examples/integrations/crawlee_example.py b/examples/integrations/crawlee_example.py new file mode 100644 index 0000000..a7543d2 --- /dev/null +++ b/examples/integrations/crawlee_example.py @@ -0,0 +1,72 @@ +"""Crawlee + CloakBrowser: stealth web crawling with PlaywrightCrawler. + +Uses a custom BrowserPlugin to swap Crawlee's default Chromium +for CloakBrowser's patched binary with source-level fingerprint patches. + +Requires: pip install cloakbrowser "crawlee[playwright]" +""" + +import asyncio + +from cloakbrowser.config import IGNORE_DEFAULT_ARGS, get_default_stealth_args +from cloakbrowser.download import ensure_binary +from typing_extensions import override + +from crawlee.browsers import ( + BrowserPool, + PlaywrightBrowserController, + PlaywrightBrowserPlugin, +) +from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext + + +class CloakBrowserPlugin(PlaywrightBrowserPlugin): + """Browser plugin that uses CloakBrowser's patched Chromium, + but otherwise keeps the functionality of PlaywrightBrowserPlugin. + """ + + @override + async def new_browser(self) -> PlaywrightBrowserController: + if not self._playwright: + raise RuntimeError('Playwright browser plugin is not initialized.') + + binary_path = ensure_binary() + stealth_args = get_default_stealth_args() + + # Merge CloakBrowser stealth args with any user-provided launch options. + launch_options = dict(self._browser_launch_options) + launch_options.pop('executable_path', None) + launch_options.pop('chromium_sandbox', None) + existing_args = list(launch_options.pop('args', [])) + launch_options['args'] = [*existing_args, *stealth_args] + + return PlaywrightBrowserController( + browser=await self._playwright.chromium.launch( + executable_path=binary_path, + ignore_default_args=IGNORE_DEFAULT_ARGS, + **launch_options, + ), + max_open_pages_per_browser=1, + # CloakBrowser handles fingerprints at the binary level. + header_generator=None, + ) + + +async def main() -> None: + crawler = PlaywrightCrawler( + max_requests_per_crawl=10, + browser_pool=BrowserPool(plugins=[CloakBrowserPlugin()]), + ) + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + title = await context.page.title() + await context.push_data({'url': context.request.url, 'title': title}) + await context.enqueue_links() + + await crawler.run(['https://example.com']) + + +if __name__ == '__main__': + asyncio.run(main())