fix(docker): add pypdf and honor stream scraping strategy

deploy/docker/api.py

@@ -865,15 +865,20 @@ async def handle_stream_crawl_request(
         # mirroring handle_crawl_request. The streaming path previously skipped
         # this, leaving /crawl/stream (and /crawl with stream=true) unguarded.
         urls = _normalize_and_validate_seeds(urls)
-        browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)
+        browser_config = BrowserConfig.load(
+            browser_config, provenance=Provenance.UNTRUSTED
+        )
         # browser_config.verbose = True # Set to False or remove for production stress testing
         browser_config.verbose = False
         from egress_broker import enforce_egress
+
         enforce_egress(browser_config)
-        crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
+        crawler_config = CrawlerRunConfig.load(
+            crawler_config, provenance=Provenance.UNTRUSTED
+        )
         from governor import clamp_deep_crawl
+
         clamp_deep_crawl(crawler_config)
-        crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
         crawler_config.stream = True
 
         # Deep crawl streaming supports exactly one start URL
@@ -941,7 +946,7 @@ async def handle_stream_crawl_request(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=str(e)
         )
-        
+
 async def handle_crawl_job(
     redis,
     background_tasks: BackgroundTasks,
@@ -1023,4 +1028,4 @@ async def _runner():
     except HTTPException:
         await redis.delete(f"task:{task_id}")
         raise
-    return {"task_id": task_id}
+    return {"task_id": task_id}

deploy/docker/requirements.txt

@@ -14,3 +14,4 @@ PyJWT==2.10.1
 mcp>=1.18.0
 websockets>=15.0.1
 httpx[http2]>=0.27.2
+pypdf>=6.0.0

tests/test_issue_2127_docker_pdf.py

@@ -0,0 +1,71 @@
+import importlib
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from packaging.requirements import InvalidRequirement, Requirement
+
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def test_default_docker_dependencies_include_pypdf():
+    lines = (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines()
+    names = set()
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith(("#", "-")):
+            continue
+        try:
+            names.add(Requirement(line).name)
+        except InvalidRequirement:
+            continue
+
+    assert "pypdf" in names
+
+
+@pytest.mark.asyncio
+async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch):
+    docker_dir = ROOT / "deploy" / "docker"
+    monkeypatch.syspath_prepend(str(docker_dir))
+
+    api = importlib.import_module("api")
+    crawler_pool = importlib.import_module("crawler_pool")
+    egress_broker = importlib.import_module("egress_broker")
+    governor = importlib.import_module("governor")
+
+    crawler = MagicMock()
+    crawler.arun_many = AsyncMock(return_value=MagicMock())
+    monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls)
+    monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None)
+    monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None)
+    monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler))
+
+    crawler_config = {
+        "type": "CrawlerRunConfig",
+        "params": {
+            "cache_mode": "bypass",
+            "stream": False,
+            "scraping_strategy": {
+                "type": "PDFContentScrapingStrategy",
+                "params": {"extract_images": False, "batch_size": 8},
+            },
+        },
+    }
+    config = {
+        "crawler": {
+            "memory_threshold_percent": 90,
+            "rate_limiter": {"base_delay": [0.1, 0.3]},
+        }
+    }
+
+    await api.handle_stream_crawl_request(
+        urls=["https://example.com/document.pdf"],
+        browser_config={"type": "BrowserConfig", "params": {}},
+        crawler_config=crawler_config,
+        config=config,
+    )
+
+    effective_config = crawler.arun_many.await_args.kwargs["config"]
+    assert isinstance(effective_config.scraping_strategy, PDFContentScrapingStrategy)