fix: CrawlResultContainer and more tests

Fix how CrawlResultContainer is used, ensuring all paths return one
and deep crawl unwraps so there's only one, not one per result.

Move it into models and directly extend CrawlResult so type hinting
works as expected for the single result.

Pass crawler into crawl_url instead of using field to simplify the code
avoiding the need for None checks.

Add more examples to AsyncWebCrawler and fix formatting to display
correctly in VS Code.

Add data checks to google_search crawler.

Correct abstract method declarations so they match implementation
which relies on yield being called.

Fix use of infinite in integer context.

Fix use of js parameter instead of js_code.

Various fixes to tests, ensuring repeatability and validation using
assert instead of just prints.

Allow pytest flags to be specified on the command line when running
a test directly with python on the cli.

Author: stevenh

crawl4ai/async_dispatcher.py

@@ -2,13 +2,16 @@
 from .async_configs import CrawlerRunConfig
 from .models import (
     CrawlResult,
+    CrawlResultContainer,
     CrawlerTaskResult,
     CrawlStatus,
     DisplayMode,
     CrawlStats,
     DomainState,
 )
 
+from .types import AsyncWebCrawler
+
 from rich.live import Live
 from rich.table import Table
 from rich.console import Console
@@ -289,7 +292,6 @@ def __init__(
         rate_limiter: Optional[RateLimiter] = None,
         monitor: Optional[CrawlerMonitor] = None,
     ):
-        self.crawler = None
         self._domain_last_hit: Dict[str, float] = {}
         self.concurrent_sessions = 0
         self.rate_limiter = rate_limiter
@@ -298,6 +300,7 @@ def __init__(
     @abstractmethod
     async def crawl_url(
         self,
+        crawler: AsyncWebCrawler,
         url: str,
         config: CrawlerRunConfig,
         task_id: str,
@@ -308,7 +311,7 @@ async def crawl_url(
     async def run_urls(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> List[CrawlerTaskResult]:
         pass
@@ -317,7 +320,7 @@ async def run_urls(
     async def run_urls_stream(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> AsyncGenerator[CrawlerTaskResult, None]:
         yield NotImplemented
@@ -342,6 +345,7 @@ def __init__(
 
     async def crawl_url(
         self,
+        crawler: AsyncWebCrawler,
         url: str,
         config: CrawlerRunConfig,
         task_id: str,
@@ -362,7 +366,7 @@ async def crawl_url(
 
             process = psutil.Process()
             start_memory = process.memory_info().rss / (1024 * 1024)
-            result = await self.crawler.arun(url, config=config, session_id=task_id)
+            result: CrawlResultContainer = await crawler.arun(url, config=config, session_id=task_id)
             end_memory = process.memory_info().rss / (1024 * 1024)
 
             memory_usage = peak_memory = end_memory - start_memory
@@ -372,7 +376,7 @@ async def crawl_url(
                     error_message = f"Rate limit retry count exceeded for domain {urlparse(url).netloc}"
                     if self.monitor:
                         self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-                    result = CrawlerTaskResult(
+                    task_result: CrawlerTaskResult = CrawlerTaskResult(
                         task_id=task_id,
                         url=url,
                         result=result,
@@ -382,8 +386,8 @@ async def crawl_url(
                         end_time=time.time(),
                         error_message=error_message,
                     )
-                    await self.result_queue.put(result)
-                    return result
+                    await self.result_queue.put(task_result)
+                    return task_result
 
             if not result.success:
                 error_message = result.error_message
@@ -396,8 +400,10 @@ async def crawl_url(
             error_message = str(e)
             if self.monitor:
                 self.monitor.update_task(task_id, status=CrawlStatus.FAILED)
-            result = CrawlResult(
-                url=url, html="", metadata={}, success=False, error_message=str(e)
+            result = CrawlResultContainer(
+                CrawlResult(
+                    url=url, html="", metadata={}, success=False, error_message=str(e)
+                )
             )
 
         finally:
@@ -420,17 +426,15 @@ async def crawl_url(
             peak_memory=peak_memory,
             start_time=start_time,
             end_time=end_time,
-            error_message=error_message,
+            error_message=error_message or "",
         )
 
     async def run_urls(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
-
         if self.monitor:
             self.monitor.start()
 
@@ -458,7 +462,7 @@ async def run_urls(
                         continue
 
                     url, task_id = task_queue.pop(0)
-                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
+                    task = asyncio.create_task(self.crawl_url(crawler, url, config, task_id))
                     active_tasks.append(task)
 
                 if not active_tasks:
@@ -480,10 +484,9 @@ async def run_urls(
     async def run_urls_stream(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> AsyncGenerator[CrawlerTaskResult, None]:
-        self.crawler = crawler
         if self.monitor:
             self.monitor.start()
 
@@ -508,7 +511,7 @@ async def run_urls_stream(
                         continue
 
                     url, task_id = task_queue.pop(0)
-                    task = asyncio.create_task(self.crawl_url(url, config, task_id))
+                    task = asyncio.create_task(self.crawl_url(crawler, url, config, task_id))
                     active_tasks.append(task)
 
                 if not active_tasks and not task_queue:
@@ -546,6 +549,7 @@ def __init__(
 
     async def crawl_url(
         self,
+        crawler: AsyncWebCrawler,
         url: str,
         config: CrawlerRunConfig,
         task_id: str,
@@ -570,7 +574,7 @@ async def crawl_url(
             async with semaphore:
                 process = psutil.Process()
                 start_memory = process.memory_info().rss / (1024 * 1024)
-                result = await self.crawler.arun(url, config=config, session_id=task_id)
+                result: CrawlResultContainer = await crawler.arun(url, config=config, session_id=task_id)
                 end_memory = process.memory_info().rss / (1024 * 1024)
 
                 memory_usage = peak_memory = end_memory - start_memory
@@ -625,16 +629,15 @@ async def crawl_url(
             peak_memory=peak_memory,
             start_time=start_time,
             end_time=end_time,
-            error_message=error_message,
+            error_message=error_message or "",
         )
 
     async def run_urls(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> List[CrawlerTaskResult]:
-        self.crawler = crawler
         if self.monitor:
             self.monitor.start()
 
@@ -647,7 +650,7 @@ async def run_urls(
                 if self.monitor:
                     self.monitor.add_task(task_id, url)
                 task = asyncio.create_task(
-                    self.crawl_url(url, config, task_id, semaphore)
+                    self.crawl_url(crawler, url, config, task_id, semaphore)
                 )
                 tasks.append(task)
 
@@ -659,10 +662,9 @@ async def run_urls(
     async def run_urls_stream(
         self,
         urls: List[str],
-        crawler: "AsyncWebCrawler",  # noqa: F821 # pyright: ignore[reportUndefinedVariable]
+        crawler: AsyncWebCrawler,
         config: CrawlerRunConfig,
     ) -> AsyncGenerator[CrawlerTaskResult, None]:
-        self.crawler = crawler
         if self.monitor:
             self.monitor.start()
 
@@ -675,7 +677,7 @@ async def run_urls_stream(
                 if self.monitor:
                     self.monitor.add_task(task_id, url)
                 task = asyncio.create_task(
-                    self.crawl_url(url, config, task_id, semaphore)
+                    self.crawl_url(crawler, url, config, task_id, semaphore)
                 )
                 tasks.append(task)