diff --git a/deploy/docker/.llm.env.example b/deploy/docker/.llm.env.example
index 2d7a2ea25..6f84f1ff1 100644
--- a/deploy/docker/.llm.env.example
+++ b/deploy/docker/.llm.env.example
@@ -1,11 +1,3 @@
-# REQUIRED for a reachable server: API token for the Docker server (0.9.0+).
-# Without it the server binds loopback inside the container and the published
-# port answers with "connection reset". Any non-empty value works, but treat it
-# as a password — use a long random string (e.g. from: openssl rand -hex 32).
-# Note: with docker compose, the token MUST be set here — exporting it in your
-# shell does not reach the container.
-CRAWL4AI_API_TOKEN=
-
# Optional: enable declarative hooks support (disabled by default)
# CRAWL4AI_HOOKS_ENABLED=true
diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md
index b895ae588..bcd2097c7 100644
--- a/deploy/docker/MIGRATION.md
+++ b/deploy/docker/MIGRATION.md
@@ -25,12 +25,13 @@ loopback by default and will not expose itself without a credential.
export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
```
-> ⚠️ **Docker Compose users:** `export` alone does **not** work — the shipped
-> `docker-compose.yml` does not forward host environment variables. Set the
-> token in the `.llm.env` file at the project root instead (the example file
-> ships an empty `CRAWL4AI_API_TOKEN=` line — fill it in).
->
-> For plain `docker run`, pass it explicitly:
+With `docker compose`, run the export in the same shell before
+`docker compose up`; the compose file passes the token into the container.
+For a persistent setup, put the `CRAWL4AI_API_TOKEN=...` line in a `.env`
+file in the project root instead — compose reads it automatically (a shell
+export still takes precedence).
+
+> ⚠️ For plain `docker run`, pass it explicitly:
> `-e CRAWL4AI_API_TOKEN="$CRAWL4AI_API_TOKEN"` (the value-less shorthand
> `-e CRAWL4AI_API_TOKEN` silently passes empty from a shell where the variable
> isn't set).
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index b624a3115..22c7483c5 100644
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -31,6 +31,8 @@ else
# No credential -> refuse to expose; serve loopback only.
GUNICORN_BIND="127.0.0.1:${PORT}"
echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
+ echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2
+ echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or 'export CRAWL4AI_API_TOKEN=...' before 'docker compose up'), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2
fi
export GUNICORN_BIND
diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py
index f066e5c88..978f21999 100644
--- a/deploy/docker/schemas.py
+++ b/deploy/docker/schemas.py
@@ -42,6 +42,14 @@ class HookConfig(BaseModel):
le=120,
description="Timeout in seconds for each hook execution",
)
+ # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an
+ # exec()-based RCE surface). Captured here (instead of being dropped by
+ # pydantic) solely so the server can tell the caller it was NOT executed;
+ # it is never run.
+ code: Optional[Any] = Field(
+ default=None,
+ description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",
+ )
class Config:
json_schema_extra = {
@@ -84,14 +92,29 @@ class ScreenshotRequest(BaseModel):
url: str
screenshot_wait_for: Optional[float] = 2
wait_for_images: Optional[bool] = False
- # output_path removed: callers never name a filesystem path (it was an
- # arbitrary-write -> RCE vector). The server writes to the sandboxed
- # artifact store and returns an opaque artifact_id.
+ # Deprecated no-op: caller paths were an arbitrary-write -> RCE vector.
+ # Never written; the server stores results in the artifact store instead.
+ output_path: Optional[str] = Field(
+ default=None,
+ deprecated=True,
+ description=(
+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+ "stored server-side; fetch via GET /artifacts/{artifact_id}."
+ ),
+ )
class PDFRequest(BaseModel):
url: str
- # output_path removed (see ScreenshotRequest).
+ # output_path deprecated no-op (see ScreenshotRequest).
+ output_path: Optional[str] = Field(
+ default=None,
+ deprecated=True,
+ description=(
+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+ "stored server-side; fetch via GET /artifacts/{artifact_id}."
+ ),
+ )
class JSEndpointRequest(BaseModel):
diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index ce1403b87..eb0fddadf 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -392,7 +392,7 @@ def _current_api_token() -> str:
app.add_middleware(
AuthGateMiddleware,
token_provider=_current_api_token,
- public_paths={HEALTH_PATH, "/token"},
+ public_paths={HEALTH_PATH, "/token", "/"},
public_prefixes=_UI_PREFIXES,
)
@@ -660,6 +660,32 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)):
# Screenshot endpoint
+_OUTPUT_PATH_WARNING = (
+ "output_path was removed in 0.9.0 and is ignored - no file was written. "
+ "The result is stored server-side; fetch it with an authenticated "
+ "GET /artifacts/{artifact_id}."
+)
+
+_HOOKS_CODE_WARNING = (
+ "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. "
+ "Use declarative hook actions instead (GET /hooks/info for the schema)."
+)
+
+
+def _reject_disabled_hooks(hooks) -> None:
+ """403 for any hooks payload while hooks are disabled. Legacy inline code
+ gets its own detail: enabling CRAWL4AI_HOOKS_ENABLED would not run it (the
+ feature was removed in 0.9.0), so the generic remedy would mislead."""
+ if hooks.code:
+ raise HTTPException(
+ 403,
+ "Inline hook code (hooks.code) was removed in 0.9.0 and cannot be "
+ "enabled; it was not executed. Use declarative hook actions instead "
+ "(GET /hooks/info), which are additionally disabled on this server "
+ "(CRAWL4AI_HOOKS_ENABLED).",
+ )
+ raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+
@app.post("/screenshot")
@limiter.limit(config["rate_limiting"]["default_limit"])
@@ -675,6 +701,9 @@ async def generate_screenshot(
sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
"""
validate_url_scheme(body.url)
+ # model_dump, not attribute access: the field is marked deprecated and
+ # reading the attribute emits DeprecationWarning on every request.
+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
crawler = None
try:
cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images)
@@ -684,7 +713,10 @@ async def generate_screenshot(
raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
screenshot_data = results[0].screenshot
art = _store_artifact("png", base64.b64decode(screenshot_data))
- return {"success": True, "screenshot": screenshot_data, **art}
+ response = {"success": True, "screenshot": screenshot_data, **art}
+ if legacy_output_path:
+ response["warning"] = _OUTPUT_PATH_WARNING
+ return response
except HTTPException:
raise
except Exception as e:
@@ -710,6 +742,9 @@ async def generate_pdf(
sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
"""
validate_url_scheme(body.url)
+ # model_dump, not attribute access: the field is marked deprecated and
+ # reading the attribute emits DeprecationWarning on every request.
+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
crawler = None
try:
cfg = CrawlerRunConfig(pdf=True)
@@ -719,7 +754,10 @@ async def generate_pdf(
raise HTTPException(500, detail=results[0].error_message or "Crawl failed")
pdf_data = results[0].pdf
art = _store_artifact("pdf", pdf_data)
- return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+ response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
+ if legacy_output_path:
+ response["warning"] = _OUTPUT_PATH_WARNING
+ return response
except HTTPException:
raise
except Exception as e:
@@ -879,7 +917,7 @@ async def crawl(
if not crawl_request.urls:
raise HTTPException(400, "At least one URL required")
if crawl_request.hooks and not HOOKS_ENABLED:
- raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+ _reject_disabled_hooks(crawl_request.hooks)
# Check whether it is a redirection for a streaming request
try:
crawler_config = CrawlerRunConfig.load(
@@ -889,7 +927,7 @@ async def crawl(
raise HTTPException(400, f"Rejected config: {e}")
if crawler_config.stream:
return await stream_process(crawl_request=crawl_request)
-
+
# Prepare hooks config if provided
hooks_config = None
if crawl_request.hooks:
@@ -897,7 +935,7 @@ async def crawl(
'hooks': crawl_request.hooks.hooks,
'timeout': crawl_request.hooks.timeout
}
-
+
results = await handle_crawl_request(
urls=crawl_request.urls,
browser_config=crawl_request.browser_config,
@@ -906,6 +944,11 @@ async def crawl(
hooks_config=hooks_config,
crawler_configs=crawl_request.crawler_configs,
)
+ if crawl_request.hooks and crawl_request.hooks.code:
+ hooks_resp = results.setdefault("hooks", {"status": "ignored", "attached": []})
+ if not crawl_request.hooks.hooks:
+ hooks_resp["status"] = "ignored"
+ hooks_resp["warning"] = _HOOKS_CODE_WARNING
return JSONResponse(results)
@@ -919,12 +962,12 @@ async def crawl_stream(
if not crawl_request.urls:
raise HTTPException(400, "At least one URL required")
if crawl_request.hooks and not HOOKS_ENABLED:
- raise HTTPException(403, "Hooks are disabled. Set CRAWL4AI_HOOKS_ENABLED=true to enable.")
+ _reject_disabled_hooks(crawl_request.hooks)
return await stream_process(crawl_request=crawl_request)
async def stream_process(crawl_request: CrawlRequestWithHooks):
-
+
# Prepare hooks config if provided# Prepare hooks config if provided
hooks_config = None
if crawl_request.hooks:
@@ -950,6 +993,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
if hooks_info:
import json
headers["X-Hooks-Status"] = json.dumps(hooks_info['status']['status'])
+ if crawl_request.hooks and crawl_request.hooks.code:
+ headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNING
return StreamingResponse(
stream_results(crawler, gen),
diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index e88c281a0..ba368b768 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -617,6 +617,25 @@ 🔥 Stress Test
}
}
+ // Build a useful error message from a failed HTTP response
+ function httpErrorMessage(response, data) {
+ let detail = data && (data.detail || data.error);
+ // FastAPI 422s send detail as an array of error objects; other
+ // non-strings would render as [object Object].
+ if (Array.isArray(detail)) {
+ detail = detail.map(e => e && e.msg ? `${(e.loc || []).join('.')}: ${e.msg}` : JSON.stringify(e)).join('; ');
+ } else if (detail && typeof detail !== 'string') {
+ detail = JSON.stringify(detail);
+ }
+ let msg = detail || `HTTP ${response.status}`;
+ if (response.status === 401) {
+ msg += getToken()
+ ? ' — token rejected; check the API token in the token bar (top right)'
+ : ' — set your API token in the token bar (top right)';
+ }
+ return msg;
+ }
+
// Generate code snippets
function generateSnippets(api, payload, method = 'POST') {
// Python snippet
@@ -747,11 +766,11 @@ 🔥 Stress Test
method: 'GET',
headers: { 'Accept': 'application/json' }
});
- responseData = await response.json();
+ responseData = await response.json().catch(() => ({}));
const time = Math.round(performance.now() - startTime);
if (!response.ok) {
updateStatus('error', time);
- throw new Error(responseData.error || 'Request failed');
+ throw new Error(httpErrorMessage(response, responseData));
}
updateStatus('success', time);
document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);
@@ -765,6 +784,12 @@ 🔥 Stress Test
body: JSON.stringify(payload)
});
+ if (!response.ok) {
+ const errData = await response.json().catch(() => ({}));
+ updateStatus('error', Math.round(performance.now() - startTime));
+ throw new Error(httpErrorMessage(response, errData));
+ }
+
const reader = response.body.getReader();
let text = '';
let maxMemory = 0;
@@ -804,12 +829,12 @@ 🔥 Stress Test
body: JSON.stringify(payload)
});
- responseData = await response.json();
+ responseData = await response.json().catch(() => ({}));
const time = Math.round(performance.now() - startTime);
if (!response.ok) {
updateStatus('error', time);
- throw new Error(responseData.error || 'Request failed');
+ throw new Error(httpErrorMessage(response, responseData));
}
updateStatus(
diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txt
index 5f7a842fe..b0206020f 100644
--- a/deploy/docker/tests/requirements.txt
+++ b/deploy/docker/tests/requirements.txt
@@ -1,2 +1,3 @@
httpx>=0.25.0
docker>=7.0.0
+pyyaml>=6.0
diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py
new file mode 100644
index 000000000..2a76d6e27
--- /dev/null
+++ b/deploy/docker/tests/test_legacy_compat.py
@@ -0,0 +1,222 @@
+"""
+Behavioral tests for 0.9.x legacy-compatibility handling:
+
+ * root redirect - "/" is public and redirects to /playground instead of
+ dying in the auth gate with a bare 401; /monitor and the
+ data routes stay gated.
+ * output_path - /screenshot and /pdf still accept the 0.8.x output_path
+ field but return a warning saying no file was written,
+ instead of silently dropping it.
+ * legacy hooks - hooks.code (removed 0.8.x inline Python) is captured,
+ never executed, and reported as status "ignored" with a
+ warning when hooks are enabled; any hooks payload is
+ still refused (403) while hooks are disabled.
+
+(The compose PID-cap check lives in test_security_container_posture.py.)
+
+These exercise the running app via TestClient (no browser / Redis needed);
+crawl internals are stubbed where a handler would otherwise need a browser.
+"""
+
+import base64
+from types import SimpleNamespace
+
+import pytest
+
+from auth import create_access_token # noqa: E402
+
+
+def _bearer() -> dict:
+ return {"Authorization": f"Bearer {create_access_token({'sub': 'user@x.com'}, scope='data')}"}
+
+
+# ───────────────────────── root redirect ─────────────────────────
+
+
+class TestRootRedirect:
+ def test_root_is_public_and_redirects_to_playground(self, stock_client):
+ r = stock_client.get("/", follow_redirects=False)
+ assert r.status_code in (302, 307), (
+ f"GET / returned {r.status_code}; expected a redirect. The auth "
+ f"gate must allow the exact path '/' so the redirect route runs."
+ )
+ assert r.headers["location"] == "/playground"
+
+ def test_monitor_and_data_routes_stay_gated(self, stock_client):
+ # /monitor must not serve content without a token; a future
+ # /monitor -> /dashboard redirect is fine (the target is UI-public),
+ # so accept 401 or a redirect, never 200.
+ r = stock_client.get("/monitor", follow_redirects=False)
+ assert r.status_code in (401, 302, 307, 308)
+ assert stock_client.get("/monitor/health").status_code == 401
+ assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401
+
+
+# ───────────────────────── output_path warning ─────────────────────────
+
+
+@pytest.fixture
+def stub_crawler(server_module, monkeypatch):
+ """Stub the crawler pool + artifact store so /screenshot and /pdf run
+ without a browser. Returns the fake artifact dict for assertions."""
+ art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1}
+ png_b64 = base64.b64encode(b"fake-png").decode()
+
+ fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf")
+
+ class _FakeCrawler:
+ async def arun(self, url, config):
+ return [fake_result]
+
+ async def fake_get_crawler(cfg):
+ return _FakeCrawler()
+
+ async def fake_release_crawler(crawler):
+ pass
+
+ monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler)
+ monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler)
+ monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art))
+ return art
+
+
+class TestOutputPathWarning:
+ @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")])
+ def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key):
+ r = stock_client.post(
+ endpoint,
+ json={"url": "https://example.com", "output_path": "/tmp/x.bin"},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ body = r.json()
+ assert body["success"] is True
+ assert "warning" in body, "output_path must not be silently dropped"
+ assert "no file was written" in body["warning"]
+ assert body["artifact_id"] == stub_crawler["artifact_id"]
+
+ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])
+ def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint):
+ r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer())
+ assert r.status_code == 200
+ assert "warning" not in r.json()
+
+ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])
+ def test_output_path_is_never_written(self, stock_client, stub_crawler, endpoint, tmp_path):
+ """Security tripwire for the 0.8.x arbitrary-write vuln: the handler
+ runs for real here (only crawler/artifact store are stubbed), so any
+ reintroduced write of body.output_path creates this file and fails."""
+ target = tmp_path / "out.bin"
+ r = stock_client.post(
+ endpoint, json={"url": "https://example.com", "output_path": str(target)},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ assert not target.exists(), "output_path must never be written to disk"
+
+
+# ───────────────────────── legacy hooks.code ─────────────────────────
+
+LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}}
+DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]}
+
+
+class TestLegacyHookCode:
+ def test_hook_config_captures_code_field(self):
+ """The legacy field must be parsed (not dropped) so it can be reported."""
+ from schemas import CrawlRequestWithHooks
+
+ req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS)
+ assert req.hooks.code == LEGACY_HOOKS["code"]
+ assert req.hooks.hooks == []
+
+ @pytest.mark.parametrize("code", [
+ "def hook(): ...", # bare string
+ {"before_goto": {"nested": "dict"}}, # non-string values
+ ["a", "list"], # wrong container entirely
+ ])
+ def test_hook_code_accepts_any_legacy_shape(self, code):
+ """The 0.8.x wire shape was never pinned; a 422 on a field we only
+ report about would be worse than the silent drop it replaces."""
+ from schemas import CrawlRequestWithHooks
+
+ req = CrawlRequestWithHooks(urls=["https://x"], hooks={"code": code})
+ assert req.hooks.code == code
+
+ # Any hooks payload is refused while hooks are disabled, but the detail
+ # must not mislead legacy callers: enabling the flag would not run
+ # hooks.code (removed in 0.9.0), so payloads carrying it get a removal
+ # message instead of the generic 'set CRAWL4AI_HOOKS_ENABLED' hint.
+ @pytest.mark.parametrize("hooks_payload,expected_detail", [
+ (LEGACY_HOOKS, "removed in 0.9.0"), # code-only
+ ({**DECLARATIVE_HOOKS, **LEGACY_HOOKS}, "removed in 0.9.0"), # mixed
+ (DECLARATIVE_HOOKS, "Set CRAWL4AI_HOOKS_ENABLED=true"), # declarative-only
+ ])
+ def test_any_hooks_payload_403_when_disabled(
+ self, server_module, monkeypatch, stock_client, hooks_payload, expected_detail
+ ):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", False)
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"], "hooks": hooks_payload},
+ headers=_bearer(),
+ )
+ assert r.status_code == 403
+ assert expected_detail in r.json()["detail"]
+
+ def _stub_crawl(self, server_module, monkeypatch, results):
+ async def fake_handle_crawl_request(**kwargs):
+ return dict(results)
+
+ monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request)
+
+ def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+ # Empty declarative specs -> api.py reports a vacuous success
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}],
+ "hooks": {"status": "success", "attached": []}},
+ )
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ hooks = r.json()["hooks"]
+ assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten"
+ assert hooks["attached"] == []
+ assert "NOT executed" in hooks["warning"]
+
+ def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client):
+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}],
+ "hooks": {"status": "success", "attached": ["before_retrieve_html"]}},
+ )
+ r = stock_client.post(
+ "/crawl",
+ json={"urls": ["https://example.com"],
+ "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ hooks = r.json()["hooks"]
+ assert hooks["status"] == "success", "real declarative execution must not be relabeled"
+ assert hooks["attached"] == ["before_retrieve_html"]
+ assert "NOT executed" in hooks["warning"]
+
+ def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client):
+ self._stub_crawl(
+ server_module, monkeypatch,
+ {"success": True, "results": [{"success": True}]},
+ )
+ r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer())
+ assert r.status_code == 200
+ assert "hooks" not in r.json()
+
+
+# The compose PID-cap check lives in
+# test_security_container_posture.py::test_pids_limit (YAML-parsed).
diff --git a/deploy/docker/tests/test_security_2026_04.py b/deploy/docker/tests/test_security_2026_04.py
index 4109de046..e015274b6 100644
--- a/deploy/docker/tests/test_security_2026_04.py
+++ b/deploy/docker/tests/test_security_2026_04.py
@@ -82,23 +82,29 @@ def validate_webhook_url(url):
# ============================================================================
class TestOutputPathRemoved(unittest.TestCase):
- """output_path is gone; the server owns paths via the artifact store.
+ """output_path never reaches the filesystem; the server owns paths via the
+ artifact store.
The old string-only validate_output_path was bypassable (symlink/TOCTOU,
sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never
- accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW,
- O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py.
+ use a caller path: output_path exists only as a deprecated no-op for 0.8.x
+ compatibility and must stay that way. Artifact-store coverage lives in
+ test_security_artifact_store.py; warning behavior in test_legacy_compat.py.
"""
- def test_screenshot_request_has_no_output_path(self):
+ def test_screenshot_output_path_is_deprecated_noop(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest
- self.assertNotIn("output_path", ScreenshotRequest.model_fields)
+ field = ScreenshotRequest.model_fields["output_path"]
+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")
+ self.assertIsNone(field.default)
- def test_pdf_request_has_no_output_path(self):
+ def test_pdf_output_path_is_deprecated_noop(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import PDFRequest
- self.assertNotIn("output_path", PDFRequest.model_fields)
+ field = PDFRequest.model_fields["output_path"]
+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")
+ self.assertIsNone(field.default)
def test_validate_output_path_deleted(self):
sys.path.insert(0, DEPLOY_DIR)
@@ -110,18 +116,21 @@ def test_validate_output_path_deleted(self):
class TestPydanticPathValidator(unittest.TestCase):
- """output_path (and its traversal validator) are gone entirely.
+ """The traversal validator is gone entirely.
Traversal rejection used to be the mitigation; the real fix is that no
- caller path is accepted at all, so there is nothing to traverse. The
- sandboxed artifact store owns all paths now.
+ caller path is ever *used*, so there is nothing to traverse. output_path
+ survives only as a deprecated no-op field (see TestOutputPathRemoved); the
+ sandboxed artifact store owns all paths.
"""
- def test_no_output_path_field_on_request_models(self):
+ def test_output_path_is_inert_on_request_models(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest, PDFRequest
- self.assertNotIn("output_path", ScreenshotRequest.model_fields)
- self.assertNotIn("output_path", PDFRequest.model_fields)
+ for model in (ScreenshotRequest, PDFRequest):
+ field = model.model_fields["output_path"]
+ self.assertTrue(field.deprecated)
+ self.assertIsNone(field.default)
def test_traversal_validator_removed(self):
# No reject_traversal validator should remain registered on the models.
diff --git a/deploy/docker/tests/test_security_container_posture.py b/deploy/docker/tests/test_security_container_posture.py
index 3399d6bd1..eb5df6153 100644
--- a/deploy/docker/tests/test_security_container_posture.py
+++ b/deploy/docker/tests/test_security_container_posture.py
@@ -104,7 +104,14 @@ def test_no_host_dev_shm_bind(self, compose):
assert "shm_size" in compose
def test_pids_limit(self, compose):
- assert "pids_limit" in compose
+ # Parse, don't grep: a raw-text search matched the word "pids_limit"
+ # inside a comment and guarded nothing. The cap lives under
+ # deploy.resources.limits (not pids_limit) for Compose v5 compatibility.
+ import yaml
+
+ base = yaml.safe_load(compose)["x-base-config"]
+ assert "pids_limit" not in base
+ assert base["deploy"]["resources"]["limits"]["pids"] == 512
def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose):
assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose
diff --git a/docker-compose.yml b/docker-compose.yml
index 1cd87ad93..6beb00169 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,13 +1,15 @@
-version: '3.8'
-
# Shared configuration for all environments
x-base-config: &base-config
ports:
- "11235:11235" # Gunicorn port
env_file:
- - .llm.env # API keys (create from .llm.env.example)
+ # API keys (create from .llm.env.example); optional so a fresh clone runs.
+ - path: .llm.env
+ required: false
+ environment:
+ # Auth token passthrough from the host shell (overwrites .llm.env).
+ - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
# Uncomment to set default environment variables (will overwrite .llm.env)
- # environment:
# - OPENAI_API_KEY=${OPENAI_API_KEY:-}
# - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY:-}
# - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
@@ -24,7 +26,6 @@ x-base-config: &base-config
- ALL
security_opt:
- no-new-privileges:true
- pids_limit: 512
# Read-only root filesystem; only these paths are writable (tmpfs).
read_only: true
tmpfs:
@@ -39,6 +40,9 @@ x-base-config: &base-config
resources:
limits:
memory: 4G
+ # PID cap; expressed here (not as pids_limit) so the file stays valid
+ # on Compose v5+, which rejects pids_limit alongside a limits block.
+ pids: 512
reservations:
memory: 1G
restart: unless-stopped
diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md
index 966672f04..09e5e37a5 100644
--- a/docs/md_v2/core/self-hosting.md
+++ b/docs/md_v2/core/self-hosting.md
@@ -62,7 +62,7 @@ When you self-host, you can scale from a single container to a full browser infr
## Prerequisites
Before we dive in, make sure you have:
-- Docker installed and running (version 20.10.0 or higher), including `docker compose` (usually bundled with Docker Desktop).
+- Docker installed and running (version 20.10.0 or higher), including `docker compose` v2.24+ (usually bundled with Docker Desktop).
- `git` for cloning the repository.
- At least 4GB of RAM available for the container (more recommended for heavy use).
- Python 3.10+ (if using the Python SDK).
@@ -212,29 +212,36 @@ cd crawl4ai
#### 2. Environment Setup (Required)
-The compose file loads `.llm.env` from the **project root directory** — the
-file must exist even if you don't use LLMs, or compose will fail with
-"env file .llm.env not found". Create it from the example and add an API token:
+Export an API token in your shell — the compose file passes it into the
+container. **Required**, or the server will be unreachable (loopback-only,
+published port → connection reset):
```bash
-# Make sure you are in the 'crawl4ai' root directory
-cp deploy/docker/.llm.env.example .llm.env
+export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
```
-Then open `.llm.env` and fill in the `CRAWL4AI_API_TOKEN=` line at the top —
-**required**, or the server will be unreachable (loopback-only). Any long
-random string works, e.g. from `openssl rand -hex 32`. One-liner:
+Prefer a file? Put the same line in a `.env` file in the project root —
+compose reads it automatically on every run, no export needed (if both are
+set, the shell export wins):
```bash
-sed -i.bak "s|^CRAWL4AI_API_TOKEN=.*|CRAWL4AI_API_TOKEN=$(openssl rand -hex 32)|" .llm.env && rm .llm.env.bak
+echo "CRAWL4AI_API_TOKEN=$(openssl rand -hex 32)" > .env
```
-Optionally add your LLM API keys in the same file.
+If you use LLMs, also create `.llm.env` in the **project root directory** with
+your API keys (optional — compose starts fine without it):
+
+```bash
+# Make sure you are in the 'crawl4ai' root directory
+cp deploy/docker/.llm.env.example .llm.env
+
+# Now edit .llm.env and add your LLM API keys
+```
-> ⚠️ **The token must go inside `.llm.env`.** `export CRAWL4AI_API_TOKEN=...`
-> in your shell does **not** work with compose — the compose file does not
-> forward host environment variables, and the server silently starts in
-> loopback-only mode (published port → connection reset).
+> ⚠️ Run the export in the **same shell** you run `docker compose up` from.
+> With compose, only the shell export or a `.env` line carries the token — a
+> `CRAWL4AI_API_TOKEN=` line in `.llm.env` is overridden by the compose
+> passthrough.
**Flexible LLM Provider Configuration:**
@@ -305,7 +312,7 @@ The `docker-compose.yml` file in the project root provides a simplified approach
> The server will be available at `http://localhost:11235` (allow ~10 seconds
> for startup). All endpoints except `GET /health` require
-> `Authorization: Bearer `.
+> `Authorization: Bearer `.
#### 4. Stopping the Service