From bc683a147f48427ba02d15b3688d5100b33ebd09 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 20 Aug 2026 21:43:41 +0530 Subject: [PATCH 1/7] docs: update system instructions to include documentation source URL for Data Commons API queries --- .../datacommons-mcp/datacommons_mcp/instructions/server.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md index cb933e3f..b3a035ff 100644 --- a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md +++ b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md @@ -1,4 +1,6 @@ -Act as a Data Commons Research Assistant. This server provides direct access to a massive, unified knowledge graph of aggregated statistical data from authoritative regional and global sources like the UN, World Bank, and Census Bureau. Use it to transform natural language queries into precise statistical insights by identifying specific indicators and retrieving their observations. It contains historical and recent data points on topics like demographics, economics, health, and environment across various geographic levels. It does not contain information on topics like real-time news, subjective viewpoints, or private corporate data. +Act as a Data Commons Research Assistant. This server provides access to a unified knowledge graph of aggregated statistical data from authoritative global sources like the UN, World Bank, and Census Bureau. Use it to answer statistical questions by identifying indicators and retrieving observations. It contains historical and recent data on demographics, economics, health, and environment across geographic levels. It does not contain real-time news, subjective viewpoints, or private corporate data. + +For Data Commons API, library, schema, or integration questions, fetch https://docs.datacommons.org/llms.txt only as needed. CRITICAL INSTRUCTION: When performing statistical research, searching for indicators, or fetching observations, you MUST first read the appropriate **MCP Resource** before calling any tools. Use your platform's standard MCP resource-reading capability to retrieve: - `skill://data-commons-researcher/SKILL.md` (for single-place queries) From 5e5ebf7a17537ecb4ccced53dff52ccbf1c1f388 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Thu, 20 Aug 2026 21:55:06 +0530 Subject: [PATCH 2/7] docs: clarify instructions for fetching official documentation versus using MCP tools for statistical research --- .../datacommons-mcp/datacommons_mcp/instructions/server.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md index b3a035ff..f8a9f3e7 100644 --- a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md +++ b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md @@ -1,12 +1,12 @@ Act as a Data Commons Research Assistant. This server provides access to a unified knowledge graph of aggregated statistical data from authoritative global sources like the UN, World Bank, and Census Bureau. Use it to answer statistical questions by identifying indicators and retrieving observations. It contains historical and recent data on demographics, economics, health, and environment across geographic levels. It does not contain real-time news, subjective viewpoints, or private corporate data. -For Data Commons API, library, schema, or integration questions, fetch https://docs.datacommons.org/llms.txt only as needed. - CRITICAL INSTRUCTION: When performing statistical research, searching for indicators, or fetching observations, you MUST first read the appropriate **MCP Resource** before calling any tools. Use your platform's standard MCP resource-reading capability to retrieve: - `skill://data-commons-researcher/SKILL.md` (for single-place queries) - `skill://data-commons-child-places-researcher/SKILL.md` (for child-places, sub-national breakdowns, or geographic hierarchies) - `skill://data-commons-multi-entity-researcher/SKILL.md` (for bilateral relationships, flows, foreign aid, trade, or multi-entity queries) +CRITICAL DOCUMENTATION INSTRUCTION: For API, library, schema, or integration questions, you MUST fetch https://docs.datacommons.org/llms.txt when needed. Use it to find relevant official documentation before answering. Do not fetch it for statistical research handled by this server's tools. + Crucially, every data point retrieved must be attributed to its original source provided in the tool output; never present statistics as "known facts" without citing the specific organization or dataset they originated from. Prioritize data integrity and transparency, ensuring that users understand both the metric and the provenance of the information provided. CRITICAL RULE ON DCIDs: Before calling any observation or metadata tool, you MUST resolve variable and place DCIDs using search tools (`search_indicators` or `search_child_indicators`) even if it means making a similar looking search call again for a place or entity you don't have the DCID for. DO NOT guess or hardcode DCIDs under ANY circumstances. Never assume a DCID based on similar looking DCIDs you have seen previously. From c25453ca25044130786497980c83c58ee278ae53 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Tue, 1 Sep 2026 17:23:40 +0530 Subject: [PATCH 3/7] feat: add optional documentation resource --- deploy/autopush.yaml | 1 + packages/datacommons-mcp/.env.sample | 4 ++ .../datacommons-mcp/datacommons_mcp/app.py | 12 ++++ .../datacommons_mcp/data_models/settings.py | 5 ++ .../datacommons_mcp/instructions/server.md | 4 +- .../datacommons-mcp/datacommons_mcp/server.py | 32 ++++++++- packages/datacommons-mcp/tests/server_test.py | 67 +++++++++++++++++++ packages/datacommons-mcp/tests/test_app.py | 19 ++++++ .../datacommons-mcp/tests/test_settings.py | 3 + 9 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 packages/datacommons-mcp/tests/server_test.py diff --git a/deploy/autopush.yaml b/deploy/autopush.yaml index 3861b4d5..7ed6eef2 100644 --- a/deploy/autopush.yaml +++ b/deploy/autopush.yaml @@ -70,6 +70,7 @@ steps: --no-allow-unauthenticated \ --set-env-vars=MCP_VERSION=$(cat _version.txt) \ --set-env-vars=DC_AGENT_API_ROOT=https://autopush.api.datacommons.org/v2 \ + --set-env-vars=DC_ENABLE_DOCUMENTATION_RESOURCE=true \ --set-env-vars=DC_API_KEY_VALIDATION_ROOT=https://autopush.api.datacommons.org \ --set-secrets=DC_API_KEY=dc-autopush-api-key-for-mcp:latest \ --service-account=datacommons-mcp-server@datcom-mixer-autopush.iam.gserviceaccount.com \ diff --git a/packages/datacommons-mcp/.env.sample b/packages/datacommons-mcp/.env.sample index 979cff87..945ef30c 100644 --- a/packages/datacommons-mcp/.env.sample +++ b/packages/datacommons-mcp/.env.sample @@ -38,6 +38,10 @@ DC_API_KEY=your_datacommons_api_key_here # - skills/{skill_name}/SKILL.md # DC_INSTRUCTIONS_DIR=/path/to/custom/instructions +# Expose the official Data Commons documentation resource (optional) +# Defaults to false. Set to true for base Data Commons deployments. +# DC_ENABLE_DOCUMENTATION_RESOURCE=true + # ============================================================================= # NON-PROD / LOCAL ROOTS (optional) # ============================================================================= diff --git a/packages/datacommons-mcp/datacommons_mcp/app.py b/packages/datacommons-mcp/datacommons_mcp/app.py index 6e64a209..d3f939ab 100644 --- a/packages/datacommons-mcp/datacommons_mcp/app.py +++ b/packages/datacommons-mcp/datacommons_mcp/app.py @@ -36,6 +36,14 @@ MCP_SERVER_NAME = "DC MCP Server" DEFAULT_INSTRUCTIONS_PACKAGE = "datacommons_mcp.instructions" SERVER_INSTRUCTIONS_FILE = "server.md" +DOCUMENTATION_RESOURCE_NAME = "data_commons_documentation_index" +DOCUMENTATION_INDEX_URI = "https://docs.datacommons.org/llms.txt" +DOCUMENTATION_ROUTING_HINT = ( + "For Data Commons API, client library, schema, dataset coverage, concept, or " + f"integration questions, read the MCP resource named `{DOCUMENTATION_RESOURCE_NAME}` " + "before searching the web. For statistical data queries, use the MCP tools and " + "skills instead." +) class DCApp: @@ -68,6 +76,10 @@ def __init__(self) -> None: # Load Server Instructions server_instructions = self._load_instructions(SERVER_INSTRUCTIONS_FILE) + if self.settings.enable_documentation_resource: + server_instructions = ( + f"{server_instructions.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" + ) @asynccontextmanager async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: diff --git a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py index 4a655300..8b1d0ad5 100644 --- a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py +++ b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py @@ -46,3 +46,8 @@ class DCSettings(BaseSettings): alias="DC_INSTRUCTIONS_DIR", description="Directory containing custom instruction files (markdown overrides)", ) + enable_documentation_resource: bool = Field( + default=False, + alias="DC_ENABLE_DOCUMENTATION_RESOURCE", + description="Expose the official Data Commons documentation resource", + ) diff --git a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md index f8a9f3e7..cb933e3f 100644 --- a/packages/datacommons-mcp/datacommons_mcp/instructions/server.md +++ b/packages/datacommons-mcp/datacommons_mcp/instructions/server.md @@ -1,12 +1,10 @@ -Act as a Data Commons Research Assistant. This server provides access to a unified knowledge graph of aggregated statistical data from authoritative global sources like the UN, World Bank, and Census Bureau. Use it to answer statistical questions by identifying indicators and retrieving observations. It contains historical and recent data on demographics, economics, health, and environment across geographic levels. It does not contain real-time news, subjective viewpoints, or private corporate data. +Act as a Data Commons Research Assistant. This server provides direct access to a massive, unified knowledge graph of aggregated statistical data from authoritative regional and global sources like the UN, World Bank, and Census Bureau. Use it to transform natural language queries into precise statistical insights by identifying specific indicators and retrieving their observations. It contains historical and recent data points on topics like demographics, economics, health, and environment across various geographic levels. It does not contain information on topics like real-time news, subjective viewpoints, or private corporate data. CRITICAL INSTRUCTION: When performing statistical research, searching for indicators, or fetching observations, you MUST first read the appropriate **MCP Resource** before calling any tools. Use your platform's standard MCP resource-reading capability to retrieve: - `skill://data-commons-researcher/SKILL.md` (for single-place queries) - `skill://data-commons-child-places-researcher/SKILL.md` (for child-places, sub-national breakdowns, or geographic hierarchies) - `skill://data-commons-multi-entity-researcher/SKILL.md` (for bilateral relationships, flows, foreign aid, trade, or multi-entity queries) -CRITICAL DOCUMENTATION INSTRUCTION: For API, library, schema, or integration questions, you MUST fetch https://docs.datacommons.org/llms.txt when needed. Use it to find relevant official documentation before answering. Do not fetch it for statistical research handled by this server's tools. - Crucially, every data point retrieved must be attributed to its original source provided in the tool output; never present statistics as "known facts" without citing the specific organization or dataset they originated from. Prioritize data integrity and transparency, ensuring that users understand both the metric and the provenance of the information provided. CRITICAL RULE ON DCIDs: Before calling any observation or metadata tool, you MUST resolve variable and place DCIDs using search tools (`search_indicators` or `search_child_indicators`) even if it means making a similar looking search call again for a place or entity you don't have the DCID for. DO NOT guess or hardcode DCIDs under ANY circumstances. Never assume a DCID based on similar looking DCIDs you have seen previously. diff --git a/packages/datacommons-mcp/datacommons_mcp/server.py b/packages/datacommons-mcp/datacommons_mcp/server.py index ad84c614..7d88d751 100644 --- a/packages/datacommons-mcp/datacommons_mcp/server.py +++ b/packages/datacommons-mcp/datacommons_mcp/server.py @@ -18,19 +18,24 @@ import logging from pathlib import Path +import requests from fastmcp import FastMCP from fastmcp.server.providers.skills import SkillsDirectoryProvider from starlette.requests import Request from starlette.responses import JSONResponse import datacommons_mcp.tools as tools -from datacommons_mcp.app import DCApp, app +from datacommons_mcp.app import ( + DOCUMENTATION_INDEX_URI, + DOCUMENTATION_RESOURCE_NAME, + DCApp, + app, +) from datacommons_mcp.version import __version__ # Configure logging logger = logging.getLogger(__name__) - # Expose the FastMCP instance for the CLI mcp = app.mcp @@ -90,5 +95,28 @@ def _register_skills(mcp_server: FastMCP, app_instance: DCApp) -> None: mcp_server.add_provider(SkillsDirectoryProvider(roots=skills_roots)) +def _register_documentation_resource(mcp_server: FastMCP, app_instance: DCApp) -> None: + """Registers the official documentation index when enabled.""" + if not app_instance.settings.enable_documentation_resource: + return + + @mcp_server.resource( + DOCUMENTATION_INDEX_URI, + name=DOCUMENTATION_RESOURCE_NAME, + title="Data Commons Documentation Index", + description=( + "Current Data Commons documentation index for API, client library, " + "schema, dataset coverage, concept, and integration questions. Use this " + "index to open only the documentation pages relevant to the question." + ), + mime_type="text/plain", + ) + def data_commons_documentation_index() -> str: + response = requests.get(DOCUMENTATION_INDEX_URI, timeout=10) + response.raise_for_status() + return response.text + + # Call provider registration on startup _register_skills(mcp, app) +_register_documentation_resource(mcp, app) diff --git a/packages/datacommons-mcp/tests/server_test.py b/packages/datacommons-mcp/tests/server_test.py new file mode 100644 index 00000000..5284ffa1 --- /dev/null +++ b/packages/datacommons-mcp/tests/server_test.py @@ -0,0 +1,67 @@ +# Copyright 2026 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for Data Commons MCP server resource registration.""" + +from types import SimpleNamespace + +import pytest +from datacommons_mcp.app import DOCUMENTATION_RESOURCE_NAME +from datacommons_mcp.server import ( + DOCUMENTATION_INDEX_URI, + _register_documentation_resource, +) +from fastmcp import Client, FastMCP + + +@pytest.mark.asyncio +async def test_documentation_resource_registered_when_enabled(requests_mock): + """The documentation resource is available when enabled.""" + documentation = "# Data Commons Documentation\n" + requests_mock.get(DOCUMENTATION_INDEX_URI, text=documentation) + mcp = FastMCP("test") + app = SimpleNamespace(settings=SimpleNamespace(enable_documentation_resource=True)) + + _register_documentation_resource(mcp, app) + + async with Client(mcp) as client: + resources = await client.list_resources() + contents = await client.read_resource(DOCUMENTATION_INDEX_URI) + + assert len(resources) == 1 + resource = resources[0] + assert str(resource.uri) == DOCUMENTATION_INDEX_URI + assert resource.name == DOCUMENTATION_RESOURCE_NAME + assert resource.title == "Data Commons Documentation Index" + assert resource.description == ( + "Current Data Commons documentation index for API, client library, " + "schema, dataset coverage, concept, and integration questions. Use this " + "index to open only the documentation pages relevant to the question." + ) + assert len(contents) == 1 + assert contents[0].text == documentation + assert requests_mock.call_count == 1 + + +@pytest.mark.asyncio +async def test_documentation_resource_not_registered_when_disabled(): + """The documentation resource can be disabled for DCP deployments.""" + mcp = FastMCP("test") + app = SimpleNamespace(settings=SimpleNamespace(enable_documentation_resource=False)) + + _register_documentation_resource(mcp, app) + + async with Client(mcp) as client: + resources = await client.list_resources() + + assert resources == [] diff --git a/packages/datacommons-mcp/tests/test_app.py b/packages/datacommons-mcp/tests/test_app.py index f0926e8c..6ab0d905 100644 --- a/packages/datacommons-mcp/tests/test_app.py +++ b/packages/datacommons-mcp/tests/test_app.py @@ -66,6 +66,25 @@ def test_app_initialization_override( assert instructions == "Custom Server Instructions" +def test_app_appends_documentation_hint_after_server_instructions( + mock_settings, mock_fastmcp, tmp_path, create_test_file +): + """Test that enabled documentation guidance follows server instructions.""" + custom_dir = tmp_path / "instructions" + create_test_file("instructions/server.md", "Custom Server Instructions") + mock_settings.return_value.instructions_dir = str(custom_dir) + mock_settings.return_value.enable_documentation_resource = True + + from datacommons_mcp.app import DOCUMENTATION_ROUTING_HINT, DCApp + + _ = DCApp() + + instructions = mock_fastmcp.call_args[1]["instructions"] + assert instructions == ( + f"Custom Server Instructions\n\n{DOCUMENTATION_ROUTING_HINT}" + ) + + def test_load_instruction_tool_override(mock_settings, tmp_path, create_test_file): """Test loading tool instructions with override.""" custom_dir = tmp_path / "instructions" diff --git a/packages/datacommons-mcp/tests/test_settings.py b/packages/datacommons-mcp/tests/test_settings.py index 97066c43..25fbed24 100644 --- a/packages/datacommons-mcp/tests/test_settings.py +++ b/packages/datacommons-mcp/tests/test_settings.py @@ -36,6 +36,7 @@ def test_loads_with_minimal_config(self): assert settings.agent_api_root == "https://api.datacommons.org/v2" assert settings.search_scope is None assert settings.instructions_dir is None + assert settings.enable_documentation_resource is False def test_loads_with_env_var_overrides(self): """Tests that environment variables override defaults for DCSettings.""" @@ -44,6 +45,7 @@ def test_loads_with_env_var_overrides(self): "DC_AGENT_API_ROOT": "https://custom-agent-api.datacommons.org/v2", "DC_SEARCH_SCOPE": "custom_only", "DC_INSTRUCTIONS_DIR": "/path/to/instructions", + "DC_ENABLE_DOCUMENTATION_RESOURCE": "true", } with patch.dict(os.environ, env_vars): settings = DCSettings() @@ -55,3 +57,4 @@ def test_loads_with_env_var_overrides(self): ) assert settings.search_scope == SearchScope.CUSTOM_ONLY assert settings.instructions_dir == "/path/to/instructions" + assert settings.enable_documentation_resource is True From f8636acb6191b1fce7c8b3ce958d4ac3bcbda23f Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Wed, 9 Sep 2026 13:52:04 +0530 Subject: [PATCH 4/7] feat: allow client overrides for documentation discovery --- packages/datacommons-mcp/README.md | 24 ++++ .../datacommons-mcp/datacommons_mcp/app.py | 23 ++-- .../datacommons_mcp/middleware.py | 80 ++++++++++++ .../datacommons-mcp/datacommons_mcp/server.py | 13 +- packages/datacommons-mcp/tests/server_test.py | 122 +++++++++++++++--- packages/datacommons-mcp/tests/test_app.py | 10 +- 6 files changed, 229 insertions(+), 43 deletions(-) diff --git a/packages/datacommons-mcp/README.md b/packages/datacommons-mcp/README.md index 1701915c..bee9145c 100644 --- a/packages/datacommons-mcp/README.md +++ b/packages/datacommons-mcp/README.md @@ -48,6 +48,30 @@ This transport mode is intended for local integrations and is programmatically c uvx datacommons-mcp serve stdio ``` +### Optional documentation resource + +`DC_ENABLE_DOCUMENTATION_RESOURCE` sets the server default (false when unset). +For stdio, pass this environment variable when launching the server process. + +HTTP clients can override the default in their MCP connection configuration: + +```json +"headers": { + "X-DC-Enable-Documentation": "true" +} +``` + +Use `"false"` to opt out even when the server default is enabled. Omit the header +to inherit the server default. Values are case-insensitive and surrounding +whitespace is ignored; other values produce an invalid-parameters error. +Send the same preference on every request and reconnect after changing it. + +When enabled, a documentation routing hint follows the existing server +instructions and the documentation index appears in resource listings. The +server fetches `llms.txt` only when the resource is read. When disabled, the +index is omitted from listings and direct reads are rejected. Existing tools +and skills are unaffected. + ## Clients You can use any MCP-enabled agent or client to connect to your running server. For example, see the [Data Commons MCP documentation](https://github.com/datacommonsorg/agent-toolkit/blob/main/docs/user_guide.md) for guides on connecting: diff --git a/packages/datacommons-mcp/datacommons_mcp/app.py b/packages/datacommons-mcp/datacommons_mcp/app.py index d3f939ab..4e7a6476 100644 --- a/packages/datacommons-mcp/datacommons_mcp/app.py +++ b/packages/datacommons-mcp/datacommons_mcp/app.py @@ -27,6 +27,7 @@ from datacommons_mcp.client import AgentAPIClient from datacommons_mcp.data_models.settings import DCSettings +from datacommons_mcp.middleware import DocumentationMiddleware from datacommons_mcp.utils import read_external_content, read_package_content from datacommons_mcp.version import __version__ @@ -36,14 +37,6 @@ MCP_SERVER_NAME = "DC MCP Server" DEFAULT_INSTRUCTIONS_PACKAGE = "datacommons_mcp.instructions" SERVER_INSTRUCTIONS_FILE = "server.md" -DOCUMENTATION_RESOURCE_NAME = "data_commons_documentation_index" -DOCUMENTATION_INDEX_URI = "https://docs.datacommons.org/llms.txt" -DOCUMENTATION_ROUTING_HINT = ( - "For Data Commons API, client library, schema, dataset coverage, concept, or " - f"integration questions, read the MCP resource named `{DOCUMENTATION_RESOURCE_NAME}` " - "before searching the web. For statistical data queries, use the MCP tools and " - "skills instead." -) class DCApp: @@ -75,11 +68,7 @@ def __init__(self) -> None: ) # Load Server Instructions - server_instructions = self._load_instructions(SERVER_INSTRUCTIONS_FILE) - if self.settings.enable_documentation_resource: - server_instructions = ( - f"{server_instructions.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" - ) + base_instructions = self._load_instructions(SERVER_INSTRUCTIONS_FILE) @asynccontextmanager async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: @@ -91,9 +80,15 @@ async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: self.mcp = FastMCP( MCP_SERVER_NAME, version=__version__, - instructions=server_instructions, + instructions=base_instructions, lifespan=lifespan, ) + self.mcp.add_middleware( + DocumentationMiddleware( + enabled=self.settings.enable_documentation_resource, + base_instructions=base_instructions, + ) + ) def _load_instructions(self, filename: str) -> str: """Loads markdown content relative to the instructions directory. diff --git a/packages/datacommons-mcp/datacommons_mcp/middleware.py b/packages/datacommons-mcp/datacommons_mcp/middleware.py index 1dbf11a7..c82e8390 100644 --- a/packages/datacommons-mcp/datacommons_mcp/middleware.py +++ b/packages/datacommons-mcp/datacommons_mcp/middleware.py @@ -1,6 +1,12 @@ import logging from collections.abc import Awaitable, Callable +from typing import Any +from fastmcp.exceptions import NotFoundError +from fastmcp.server.dependencies import get_http_headers +from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext +from mcp import McpError +from mcp.types import INVALID_PARAMS, ErrorData from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import Response @@ -9,6 +15,80 @@ logger = logging.getLogger(__name__) +DOCUMENTATION_HEADER = "X-DC-Enable-Documentation" +DOCUMENTATION_RESOURCE_NAME = "data_commons_documentation_index" +DOCUMENTATION_INDEX_URI = "https://docs.datacommons.org/llms.txt" +DOCUMENTATION_ROUTING_HINT = ( + "For Data Commons API, client library, schema, dataset coverage, concept, or " + f"integration questions, read the MCP resource named `{DOCUMENTATION_RESOURCE_NAME}` " + "before searching the web. For statistical data queries, use the MCP tools and " + "skills instead." +) + + +class DocumentationMiddleware(Middleware): + """Apply the client's documentation preference, falling back to the server.""" + + def __init__(self, *, enabled: bool, base_instructions: str) -> None: + self._default_enabled = enabled + self._base_instructions = base_instructions + self._documentation_instructions = ( + f"{base_instructions.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" + ) + + def _enabled(self) -> bool: + value = get_http_headers().get(DOCUMENTATION_HEADER.lower()) + if value is None: + return self._default_enabled + value = value.strip().lower() + if value not in ("true", "false"): + raise McpError( + ErrorData( + code=INVALID_PARAMS, + message=f"{DOCUMENTATION_HEADER} must be true or false", + ) + ) + return value == "true" + + async def on_request(self, context: MiddlewareContext, call_next: CallNext) -> Any: # noqa: ANN401 + self._enabled() # Validate the preference on every request. + return await call_next(context) + + async def on_initialize( + self, context: MiddlewareContext, call_next: CallNext + ) -> Any: # noqa: ANN401 + instructions = ( + self._documentation_instructions + if self._enabled() + else self._base_instructions + ) + session = context.fastmcp_context.session + # FastMCP 3.4.2 sends initialization before middleware returns. Copy the + # session options before dispatch; never mutate the shared options. + session._init_options = session._init_options.model_copy( + update={"instructions": instructions} + ) + return await call_next(context) + + async def on_list_resources( + self, context: MiddlewareContext, call_next: CallNext + ) -> Any: # noqa: ANN401 + resources = await call_next(context) + if self._enabled(): + return resources + return [ + resource + for resource in resources + if str(resource.uri) != DOCUMENTATION_INDEX_URI + ] + + async def on_read_resource( + self, context: MiddlewareContext, call_next: CallNext + ) -> Any: # noqa: ANN401 + if str(context.message.uri) == DOCUMENTATION_INDEX_URI and not self._enabled(): + raise NotFoundError(f"Unknown resource: {DOCUMENTATION_INDEX_URI}") + return await call_next(context) + class APIKeyMiddleware(BaseHTTPMiddleware): """Middleware to extract X-API-Key header and set it as the override API key diff --git a/packages/datacommons-mcp/datacommons_mcp/server.py b/packages/datacommons-mcp/datacommons_mcp/server.py index 7d88d751..8a6c28b1 100644 --- a/packages/datacommons-mcp/datacommons_mcp/server.py +++ b/packages/datacommons-mcp/datacommons_mcp/server.py @@ -25,11 +25,10 @@ from starlette.responses import JSONResponse import datacommons_mcp.tools as tools -from datacommons_mcp.app import ( +from datacommons_mcp.app import DCApp, app +from datacommons_mcp.middleware import ( DOCUMENTATION_INDEX_URI, DOCUMENTATION_RESOURCE_NAME, - DCApp, - app, ) from datacommons_mcp.version import __version__ @@ -95,10 +94,8 @@ def _register_skills(mcp_server: FastMCP, app_instance: DCApp) -> None: mcp_server.add_provider(SkillsDirectoryProvider(roots=skills_roots)) -def _register_documentation_resource(mcp_server: FastMCP, app_instance: DCApp) -> None: - """Registers the official documentation index when enabled.""" - if not app_instance.settings.enable_documentation_resource: - return +def _register_documentation_resource(mcp_server: FastMCP) -> None: + """Register the index; middleware controls client access.""" @mcp_server.resource( DOCUMENTATION_INDEX_URI, @@ -119,4 +116,4 @@ def data_commons_documentation_index() -> str: # Call provider registration on startup _register_skills(mcp, app) -_register_documentation_resource(mcp, app) +_register_documentation_resource(mcp) diff --git a/packages/datacommons-mcp/tests/server_test.py b/packages/datacommons-mcp/tests/server_test.py index 5284ffa1..8eb6a8b6 100644 --- a/packages/datacommons-mcp/tests/server_test.py +++ b/packages/datacommons-mcp/tests/server_test.py @@ -13,15 +13,40 @@ # limitations under the License. """Tests for Data Commons MCP server resource registration.""" -from types import SimpleNamespace +import asyncio +from functools import partial import pytest -from datacommons_mcp.app import DOCUMENTATION_RESOURCE_NAME -from datacommons_mcp.server import ( +from datacommons_mcp.middleware import ( DOCUMENTATION_INDEX_URI, - _register_documentation_resource, + DOCUMENTATION_RESOURCE_NAME, + DOCUMENTATION_ROUTING_HINT, + DocumentationMiddleware, ) +from datacommons_mcp.server import _register_documentation_resource from fastmcp import Client, FastMCP +from fastmcp.client.transports import StreamableHttpTransport +from fastmcp.exceptions import ResourceError +from fastmcp.utilities.tests import run_server_async +from mcp import McpError + + +def documentation_server(enabled): + base = "Custom server instructions.\n" + server = FastMCP("test", instructions=base) + server.add_middleware( + DocumentationMiddleware( + enabled=enabled, + base_instructions=base, + ) + ) + _register_documentation_resource(server) + + @server.resource("test://unrelated") + def unrelated(): + return "unchanged" + + return server @pytest.mark.asyncio @@ -29,17 +54,13 @@ async def test_documentation_resource_registered_when_enabled(requests_mock): """The documentation resource is available when enabled.""" documentation = "# Data Commons Documentation\n" requests_mock.get(DOCUMENTATION_INDEX_URI, text=documentation) - mcp = FastMCP("test") - app = SimpleNamespace(settings=SimpleNamespace(enable_documentation_resource=True)) - - _register_documentation_resource(mcp, app) + mcp = documentation_server(enabled=True) async with Client(mcp) as client: resources = await client.list_resources() contents = await client.read_resource(DOCUMENTATION_INDEX_URI) - assert len(resources) == 1 - resource = resources[0] + resource = next(r for r in resources if str(r.uri) == DOCUMENTATION_INDEX_URI) assert str(resource.uri) == DOCUMENTATION_INDEX_URI assert resource.name == DOCUMENTATION_RESOURCE_NAME assert resource.title == "Data Commons Documentation Index" @@ -56,12 +77,83 @@ async def test_documentation_resource_registered_when_enabled(requests_mock): @pytest.mark.asyncio async def test_documentation_resource_not_registered_when_disabled(): """The documentation resource can be disabled for DCP deployments.""" - mcp = FastMCP("test") - app = SimpleNamespace(settings=SimpleNamespace(enable_documentation_resource=False)) - - _register_documentation_resource(mcp, app) + mcp = documentation_server(enabled=False) async with Client(mcp) as client: resources = await client.list_resources() - assert resources == [] + assert [str(r.uri) for r in resources] == ["test://unrelated"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("default", [False, True]) +async def test_documentation_http_overrides(default, monkeypatch, requests_mock): + server = documentation_server(default) + original = server.instructions + monkeypatch.setattr( + server, "run_http_async", partial(server.run_http_async, stateless_http=True) + ) + fetch = requests_mock.get(DOCUMENTATION_INDEX_URI, text="# Documentation") + + async with run_server_async(server) as url: + # Initialization and listing must never download the index. + async def check(header): + enabled = default if header is None else header.lower() == "true" + headers = {} if header is None else {"X-DC-Enable-Documentation": header} + async with Client(StreamableHttpTransport(url, headers=headers)) as client: + result = await client.initialize() + expected = "Custom server instructions.\n" + if enabled: + expected = f"{expected.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" + assert result.instructions == expected + for _ in range(2): + resources = await client.list_resources() + assert ( + any(str(r.uri) == DOCUMENTATION_INDEX_URI for r in resources) + == enabled + ) + assert (await client.read_resource("test://unrelated"))[ + 0 + ].text == "unchanged" + + await asyncio.gather( + *(check(value) for value in [None, "true", "false", "TRUE"]) + ) + assert fetch.call_count == 0 + for header in [None, "true", "false"]: + enabled = default if header is None else header == "true" + headers = {} if header is None else {"X-DC-Enable-Documentation": header} + async with Client(StreamableHttpTransport(url, headers=headers)) as client: + if enabled: + assert (await client.read_resource(DOCUMENTATION_INDEX_URI))[ + 0 + ].text == "# Documentation" + else: + with pytest.raises( + (ResourceError, McpError), match="Unknown resource" + ): + await client.read_resource(DOCUMENTATION_INDEX_URI) + assert fetch.call_count == 1 + int(default) + async with Client( + StreamableHttpTransport( + url, headers={"X-DC-Enable-Documentation": "invalid"} + ), + auto_initialize=False, + ) as client: + with pytest.raises(McpError, match="must be true or false") as error: + await client.initialize() + assert error.value.error.code == -32602 + assert server.instructions == original + + +@pytest.mark.parametrize(("value", "expected"), [(" TRUE ", True), (" false ", False)]) +def test_documentation_header_whitespace(value, expected, monkeypatch): + monkeypatch.setattr( + "datacommons_mcp.middleware.get_http_headers", + lambda: {"x-dc-enable-documentation": value}, + ) + middleware = DocumentationMiddleware( + enabled=False, + base_instructions="base", + ) + assert middleware._enabled() is expected diff --git a/packages/datacommons-mcp/tests/test_app.py b/packages/datacommons-mcp/tests/test_app.py index 6ab0d905..a083b4d7 100644 --- a/packages/datacommons-mcp/tests/test_app.py +++ b/packages/datacommons-mcp/tests/test_app.py @@ -66,23 +66,21 @@ def test_app_initialization_override( assert instructions == "Custom Server Instructions" -def test_app_appends_documentation_hint_after_server_instructions( +def test_app_leaves_documentation_hint_to_middleware( mock_settings, mock_fastmcp, tmp_path, create_test_file ): - """Test that enabled documentation guidance follows server instructions.""" + """The shared server instructions remain unchanged when docs are enabled.""" custom_dir = tmp_path / "instructions" create_test_file("instructions/server.md", "Custom Server Instructions") mock_settings.return_value.instructions_dir = str(custom_dir) mock_settings.return_value.enable_documentation_resource = True - from datacommons_mcp.app import DOCUMENTATION_ROUTING_HINT, DCApp + from datacommons_mcp.app import DCApp _ = DCApp() instructions = mock_fastmcp.call_args[1]["instructions"] - assert instructions == ( - f"Custom Server Instructions\n\n{DOCUMENTATION_ROUTING_HINT}" - ) + assert instructions == "Custom Server Instructions" def test_load_instruction_tool_override(mock_settings, tmp_path, create_test_file): From a3c45e68f56e71f6489591c07353c5d4af85f4ba Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Wed, 9 Sep 2026 13:59:59 +0530 Subject: [PATCH 5/7] chore: remove DC_ENABLE_DOCUMENTATION_RESOURCE env var from autopush deployment --- deploy/autopush.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/autopush.yaml b/deploy/autopush.yaml index 7ed6eef2..3861b4d5 100644 --- a/deploy/autopush.yaml +++ b/deploy/autopush.yaml @@ -70,7 +70,6 @@ steps: --no-allow-unauthenticated \ --set-env-vars=MCP_VERSION=$(cat _version.txt) \ --set-env-vars=DC_AGENT_API_ROOT=https://autopush.api.datacommons.org/v2 \ - --set-env-vars=DC_ENABLE_DOCUMENTATION_RESOURCE=true \ --set-env-vars=DC_API_KEY_VALIDATION_ROOT=https://autopush.api.datacommons.org \ --set-secrets=DC_API_KEY=dc-autopush-api-key-for-mcp:latest \ --service-account=datacommons-mcp-server@datcom-mixer-autopush.iam.gserviceaccount.com \ From e9bca4dfb79298b22004664c1ad26f99ea29097d Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Mon, 14 Sep 2026 18:47:23 +0530 Subject: [PATCH 6/7] Simplify optional documentation guidance to a direct URL --- packages/datacommons-mcp/README.md | 16 ++- .../datacommons-mcp/datacommons_mcp/app.py | 10 +- .../datacommons_mcp/data_models/settings.py | 6 +- .../doc_instructions_extension.md | 1 + .../datacommons_mcp/middleware.py | 36 +---- .../datacommons-mcp/datacommons_mcp/server.py | 26 ---- packages/datacommons-mcp/pyproject.toml | 1 + packages/datacommons-mcp/tests/server_test.py | 135 ++++++------------ packages/datacommons-mcp/tests/test_app.py | 13 +- .../datacommons-mcp/tests/test_settings.py | 6 +- 10 files changed, 86 insertions(+), 164 deletions(-) create mode 100644 packages/datacommons-mcp/datacommons_mcp/instructions/doc_instructions_extension.md diff --git a/packages/datacommons-mcp/README.md b/packages/datacommons-mcp/README.md index bee9145c..224b1c5a 100644 --- a/packages/datacommons-mcp/README.md +++ b/packages/datacommons-mcp/README.md @@ -48,9 +48,9 @@ This transport mode is intended for local integrations and is programmatically c uvx datacommons-mcp serve stdio ``` -### Optional documentation resource +### Optional documentation guidance -`DC_ENABLE_DOCUMENTATION_RESOURCE` sets the server default (false when unset). +`DC_ENABLE_DOCUMENTATION` sets the server default (false when unset). For stdio, pass this environment variable when launching the server process. HTTP clients can override the default in their MCP connection configuration: @@ -67,11 +67,17 @@ whitespace is ignored; other values produce an invalid-parameters error. Send the same preference on every request and reconnect after changing it. When enabled, a documentation routing hint follows the existing server -instructions and the documentation index appears in resource listings. The -server fetches `llms.txt` only when the resource is read. When disabled, the -index is omitted from listings and direct reads are rejected. Existing tools +instructions and directs the client to fetch https://docs.datacommons.org/llms.txt +for relevant documentation questions. The client needs URL-reading capability +and permission to access the index and linked pages. The server does not fetch +the index or expose it as an MCP resource. When disabled, no documentation hint +is supplied; this does not prevent access to the public URL. Existing tools and skills are unaffected. +To customize the hint, place `doc_instructions_extension.md` in +`DC_INSTRUCTIONS_DIR`. It uses the same override mechanism as `server.md`, +which remains separate and always precedes the extension. + ## Clients You can use any MCP-enabled agent or client to connect to your running server. For example, see the [Data Commons MCP documentation](https://github.com/datacommonsorg/agent-toolkit/blob/main/docs/user_guide.md) for guides on connecting: diff --git a/packages/datacommons-mcp/datacommons_mcp/app.py b/packages/datacommons-mcp/datacommons_mcp/app.py index 4e7a6476..b79968f5 100644 --- a/packages/datacommons-mcp/datacommons_mcp/app.py +++ b/packages/datacommons-mcp/datacommons_mcp/app.py @@ -37,6 +37,7 @@ MCP_SERVER_NAME = "DC MCP Server" DEFAULT_INSTRUCTIONS_PACKAGE = "datacommons_mcp.instructions" SERVER_INSTRUCTIONS_FILE = "server.md" +DOCUMENTATION_INSTRUCTIONS_FILE = "doc_instructions_extension.md" class DCApp: @@ -69,6 +70,12 @@ def __init__(self) -> None: # Load Server Instructions base_instructions = self._load_instructions(SERVER_INSTRUCTIONS_FILE) + documentation_extension = self._load_instructions( + DOCUMENTATION_INSTRUCTIONS_FILE + ) + documentation_instructions = ( + f"{base_instructions.rstrip()}\n\n{documentation_extension}" + ) @asynccontextmanager async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: @@ -85,8 +92,9 @@ async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: ) self.mcp.add_middleware( DocumentationMiddleware( - enabled=self.settings.enable_documentation_resource, + enabled=self.settings.enable_documentation, base_instructions=base_instructions, + documentation_instructions=documentation_instructions, ) ) diff --git a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py index 8b1d0ad5..6a38bf27 100644 --- a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py +++ b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py @@ -46,8 +46,8 @@ class DCSettings(BaseSettings): alias="DC_INSTRUCTIONS_DIR", description="Directory containing custom instruction files (markdown overrides)", ) - enable_documentation_resource: bool = Field( + enable_documentation: bool = Field( default=False, - alias="DC_ENABLE_DOCUMENTATION_RESOURCE", - description="Expose the official Data Commons documentation resource", + alias="DC_ENABLE_DOCUMENTATION", + description="Include guidance for accessing official Data Commons documentation", ) diff --git a/packages/datacommons-mcp/datacommons_mcp/instructions/doc_instructions_extension.md b/packages/datacommons-mcp/datacommons_mcp/instructions/doc_instructions_extension.md new file mode 100644 index 00000000..0afdb175 --- /dev/null +++ b/packages/datacommons-mcp/datacommons_mcp/instructions/doc_instructions_extension.md @@ -0,0 +1 @@ +For Data Commons API, client library, schema, dataset coverage, concept, or integration questions, fetch https://docs.datacommons.org/llms.txt and use its index to open only the documentation pages relevant to the question. For statistical data queries, use the MCP tools and skills instead. diff --git a/packages/datacommons-mcp/datacommons_mcp/middleware.py b/packages/datacommons-mcp/datacommons_mcp/middleware.py index c82e8390..2750569d 100644 --- a/packages/datacommons-mcp/datacommons_mcp/middleware.py +++ b/packages/datacommons-mcp/datacommons_mcp/middleware.py @@ -2,7 +2,6 @@ from collections.abc import Awaitable, Callable from typing import Any -from fastmcp.exceptions import NotFoundError from fastmcp.server.dependencies import get_http_headers from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext from mcp import McpError @@ -16,25 +15,17 @@ logger = logging.getLogger(__name__) DOCUMENTATION_HEADER = "X-DC-Enable-Documentation" -DOCUMENTATION_RESOURCE_NAME = "data_commons_documentation_index" -DOCUMENTATION_INDEX_URI = "https://docs.datacommons.org/llms.txt" -DOCUMENTATION_ROUTING_HINT = ( - "For Data Commons API, client library, schema, dataset coverage, concept, or " - f"integration questions, read the MCP resource named `{DOCUMENTATION_RESOURCE_NAME}` " - "before searching the web. For statistical data queries, use the MCP tools and " - "skills instead." -) class DocumentationMiddleware(Middleware): """Apply the client's documentation preference, falling back to the server.""" - def __init__(self, *, enabled: bool, base_instructions: str) -> None: + def __init__( + self, *, enabled: bool, base_instructions: str, documentation_instructions: str + ) -> None: self._default_enabled = enabled self._base_instructions = base_instructions - self._documentation_instructions = ( - f"{base_instructions.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" - ) + self._documentation_instructions = documentation_instructions def _enabled(self) -> bool: value = get_http_headers().get(DOCUMENTATION_HEADER.lower()) @@ -70,25 +61,6 @@ async def on_initialize( ) return await call_next(context) - async def on_list_resources( - self, context: MiddlewareContext, call_next: CallNext - ) -> Any: # noqa: ANN401 - resources = await call_next(context) - if self._enabled(): - return resources - return [ - resource - for resource in resources - if str(resource.uri) != DOCUMENTATION_INDEX_URI - ] - - async def on_read_resource( - self, context: MiddlewareContext, call_next: CallNext - ) -> Any: # noqa: ANN401 - if str(context.message.uri) == DOCUMENTATION_INDEX_URI and not self._enabled(): - raise NotFoundError(f"Unknown resource: {DOCUMENTATION_INDEX_URI}") - return await call_next(context) - class APIKeyMiddleware(BaseHTTPMiddleware): """Middleware to extract X-API-Key header and set it as the override API key diff --git a/packages/datacommons-mcp/datacommons_mcp/server.py b/packages/datacommons-mcp/datacommons_mcp/server.py index 8a6c28b1..b6f41a4c 100644 --- a/packages/datacommons-mcp/datacommons_mcp/server.py +++ b/packages/datacommons-mcp/datacommons_mcp/server.py @@ -18,7 +18,6 @@ import logging from pathlib import Path -import requests from fastmcp import FastMCP from fastmcp.server.providers.skills import SkillsDirectoryProvider from starlette.requests import Request @@ -26,10 +25,6 @@ import datacommons_mcp.tools as tools from datacommons_mcp.app import DCApp, app -from datacommons_mcp.middleware import ( - DOCUMENTATION_INDEX_URI, - DOCUMENTATION_RESOURCE_NAME, -) from datacommons_mcp.version import __version__ # Configure logging @@ -94,26 +89,5 @@ def _register_skills(mcp_server: FastMCP, app_instance: DCApp) -> None: mcp_server.add_provider(SkillsDirectoryProvider(roots=skills_roots)) -def _register_documentation_resource(mcp_server: FastMCP) -> None: - """Register the index; middleware controls client access.""" - - @mcp_server.resource( - DOCUMENTATION_INDEX_URI, - name=DOCUMENTATION_RESOURCE_NAME, - title="Data Commons Documentation Index", - description=( - "Current Data Commons documentation index for API, client library, " - "schema, dataset coverage, concept, and integration questions. Use this " - "index to open only the documentation pages relevant to the question." - ), - mime_type="text/plain", - ) - def data_commons_documentation_index() -> str: - response = requests.get(DOCUMENTATION_INDEX_URI, timeout=10) - response.raise_for_status() - return response.text - - # Call provider registration on startup _register_skills(mcp, app) -_register_documentation_resource(mcp) diff --git a/packages/datacommons-mcp/pyproject.toml b/packages/datacommons-mcp/pyproject.toml index c8ea58fe..24c9f455 100644 --- a/packages/datacommons-mcp/pyproject.toml +++ b/packages/datacommons-mcp/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.11, <3.14" dependencies = [ "fastapi>=0.115.0", "uvicorn", + # Initialization middleware uses private session options; revalidate before upgrading. "fastmcp==3.4.2", "requests", "pydantic>=2.11.7", diff --git a/packages/datacommons-mcp/tests/server_test.py b/packages/datacommons-mcp/tests/server_test.py index 8eb6a8b6..e2b16bcf 100644 --- a/packages/datacommons-mcp/tests/server_test.py +++ b/packages/datacommons-mcp/tests/server_test.py @@ -11,107 +11,59 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for Data Commons MCP server resource registration.""" +"""Protocol tests for client-controlled documentation guidance.""" import asyncio from functools import partial import pytest -from datacommons_mcp.middleware import ( - DOCUMENTATION_INDEX_URI, - DOCUMENTATION_RESOURCE_NAME, - DOCUMENTATION_ROUTING_HINT, - DocumentationMiddleware, -) -from datacommons_mcp.server import _register_documentation_resource -from fastmcp import Client, FastMCP +from datacommons_mcp.app import DCApp +from datacommons_mcp.middleware import DOCUMENTATION_HEADER, DocumentationMiddleware +from fastmcp import Client from fastmcp.client.transports import StreamableHttpTransport -from fastmcp.exceptions import ResourceError from fastmcp.utilities.tests import run_server_async from mcp import McpError -def documentation_server(enabled): - base = "Custom server instructions.\n" - server = FastMCP("test", instructions=base) - server.add_middleware( - DocumentationMiddleware( - enabled=enabled, - base_instructions=base, - ) - ) - _register_documentation_resource(server) - - @server.resource("test://unrelated") - def unrelated(): - return "unchanged" - - return server - - -@pytest.mark.asyncio -async def test_documentation_resource_registered_when_enabled(requests_mock): - """The documentation resource is available when enabled.""" - documentation = "# Data Commons Documentation\n" - requests_mock.get(DOCUMENTATION_INDEX_URI, text=documentation) - mcp = documentation_server(enabled=True) - - async with Client(mcp) as client: - resources = await client.list_resources() - contents = await client.read_resource(DOCUMENTATION_INDEX_URI) - - resource = next(r for r in resources if str(r.uri) == DOCUMENTATION_INDEX_URI) - assert str(resource.uri) == DOCUMENTATION_INDEX_URI - assert resource.name == DOCUMENTATION_RESOURCE_NAME - assert resource.title == "Data Commons Documentation Index" - assert resource.description == ( - "Current Data Commons documentation index for API, client library, " - "schema, dataset coverage, concept, and integration questions. Use this " - "index to open only the documentation pages relevant to the question." - ) - assert len(contents) == 1 - assert contents[0].text == documentation - assert requests_mock.call_count == 1 - - -@pytest.mark.asyncio -async def test_documentation_resource_not_registered_when_disabled(): - """The documentation resource can be disabled for DCP deployments.""" - mcp = documentation_server(enabled=False) - - async with Client(mcp) as client: - resources = await client.list_resources() - - assert [str(r.uri) for r in resources] == ["test://unrelated"] +@pytest.fixture +def documentation_app(monkeypatch, tmp_path, create_test_file): + create_test_file("server.md", "Custom server instructions.\n") + create_test_file("doc_instructions_extension.md", "Custom documentation hint.\n") + monkeypatch.setenv("DC_INSTRUCTIONS_DIR", str(tmp_path)) + return DCApp @pytest.mark.asyncio @pytest.mark.parametrize("default", [False, True]) -async def test_documentation_http_overrides(default, monkeypatch, requests_mock): - server = documentation_server(default) +async def test_documentation_http_overrides(default, documentation_app, monkeypatch): + monkeypatch.setenv("DC_ENABLE_DOCUMENTATION", str(default)) + app = documentation_app() + server = app.mcp original = server.instructions monkeypatch.setattr( server, "run_http_async", partial(server.run_http_async, stateless_http=True) ) - fetch = requests_mock.get(DOCUMENTATION_INDEX_URI, text="# Documentation") + + @server.resource("test://unrelated") + def unrelated(): + return "unchanged" async with run_server_async(server) as url: - # Initialization and listing must never download the index. + async def check(header): enabled = default if header is None else header.lower() == "true" - headers = {} if header is None else {"X-DC-Enable-Documentation": header} + headers = {} if header is None else {DOCUMENTATION_HEADER: header} async with Client(StreamableHttpTransport(url, headers=headers)) as client: result = await client.initialize() expected = "Custom server instructions.\n" if enabled: - expected = f"{expected.rstrip()}\n\n{DOCUMENTATION_ROUTING_HINT}" + expected = ( + "Custom server instructions.\n\nCustom documentation hint.\n" + ) assert result.instructions == expected for _ in range(2): resources = await client.list_resources() - assert ( - any(str(r.uri) == DOCUMENTATION_INDEX_URI for r in resources) - == enabled - ) + assert [str(r.uri) for r in resources] == ["test://unrelated"] assert (await client.read_resource("test://unrelated"))[ 0 ].text == "unchanged" @@ -119,25 +71,8 @@ async def check(header): await asyncio.gather( *(check(value) for value in [None, "true", "false", "TRUE"]) ) - assert fetch.call_count == 0 - for header in [None, "true", "false"]: - enabled = default if header is None else header == "true" - headers = {} if header is None else {"X-DC-Enable-Documentation": header} - async with Client(StreamableHttpTransport(url, headers=headers)) as client: - if enabled: - assert (await client.read_resource(DOCUMENTATION_INDEX_URI))[ - 0 - ].text == "# Documentation" - else: - with pytest.raises( - (ResourceError, McpError), match="Unknown resource" - ): - await client.read_resource(DOCUMENTATION_INDEX_URI) - assert fetch.call_count == 1 + int(default) async with Client( - StreamableHttpTransport( - url, headers={"X-DC-Enable-Documentation": "invalid"} - ), + StreamableHttpTransport(url, headers={DOCUMENTATION_HEADER: "invalid"}), auto_initialize=False, ) as client: with pytest.raises(McpError, match="must be true or false") as error: @@ -146,14 +81,32 @@ async def check(header): assert server.instructions == original +@pytest.mark.asyncio +@pytest.mark.parametrize("default", [None, "false", "true"]) +async def test_documentation_without_http_header( + default, documentation_app, monkeypatch +): + """Without HTTP headers, the environment controls the instructions as for stdio.""" + if default is not None: + monkeypatch.setenv("DC_ENABLE_DOCUMENTATION", default) + app = documentation_app() + async with Client(app.mcp) as client: + result = await client.initialize() + expected = "Custom server instructions.\n" + if default == "true": + expected = "Custom server instructions.\n\nCustom documentation hint.\n" + assert result.instructions == expected + + @pytest.mark.parametrize(("value", "expected"), [(" TRUE ", True), (" false ", False)]) def test_documentation_header_whitespace(value, expected, monkeypatch): monkeypatch.setattr( "datacommons_mcp.middleware.get_http_headers", - lambda: {"x-dc-enable-documentation": value}, + lambda: {DOCUMENTATION_HEADER.lower(): value}, ) middleware = DocumentationMiddleware( enabled=False, base_instructions="base", + documentation_instructions="base with documentation", ) assert middleware._enabled() is expected diff --git a/packages/datacommons-mcp/tests/test_app.py b/packages/datacommons-mcp/tests/test_app.py index a083b4d7..07368dcf 100644 --- a/packages/datacommons-mcp/tests/test_app.py +++ b/packages/datacommons-mcp/tests/test_app.py @@ -66,14 +66,14 @@ def test_app_initialization_override( assert instructions == "Custom Server Instructions" -def test_app_leaves_documentation_hint_to_middleware( +def test_app_prepares_documentation_instructions( mock_settings, mock_fastmcp, tmp_path, create_test_file ): - """The shared server instructions remain unchanged when docs are enabled.""" + """Prepare the packaged extension while leaving shared instructions unchanged.""" custom_dir = tmp_path / "instructions" create_test_file("instructions/server.md", "Custom Server Instructions") mock_settings.return_value.instructions_dir = str(custom_dir) - mock_settings.return_value.enable_documentation_resource = True + mock_settings.return_value.enable_documentation = True from datacommons_mcp.app import DCApp @@ -81,6 +81,13 @@ def test_app_leaves_documentation_hint_to_middleware( instructions = mock_fastmcp.call_args[1]["instructions"] assert instructions == "Custom Server Instructions" + middleware = mock_fastmcp.return_value.add_middleware.call_args[0][0] + assert middleware._base_instructions == instructions + assert ( + "https://docs.datacommons.org/llms.txt" + in middleware._documentation_instructions + ) + assert middleware._documentation_instructions.startswith(f"{instructions}\n\n") def test_load_instruction_tool_override(mock_settings, tmp_path, create_test_file): diff --git a/packages/datacommons-mcp/tests/test_settings.py b/packages/datacommons-mcp/tests/test_settings.py index 25fbed24..fcdbff22 100644 --- a/packages/datacommons-mcp/tests/test_settings.py +++ b/packages/datacommons-mcp/tests/test_settings.py @@ -36,7 +36,7 @@ def test_loads_with_minimal_config(self): assert settings.agent_api_root == "https://api.datacommons.org/v2" assert settings.search_scope is None assert settings.instructions_dir is None - assert settings.enable_documentation_resource is False + assert settings.enable_documentation is False def test_loads_with_env_var_overrides(self): """Tests that environment variables override defaults for DCSettings.""" @@ -45,7 +45,7 @@ def test_loads_with_env_var_overrides(self): "DC_AGENT_API_ROOT": "https://custom-agent-api.datacommons.org/v2", "DC_SEARCH_SCOPE": "custom_only", "DC_INSTRUCTIONS_DIR": "/path/to/instructions", - "DC_ENABLE_DOCUMENTATION_RESOURCE": "true", + "DC_ENABLE_DOCUMENTATION": "true", } with patch.dict(os.environ, env_vars): settings = DCSettings() @@ -57,4 +57,4 @@ def test_loads_with_env_var_overrides(self): ) assert settings.search_scope == SearchScope.CUSTOM_ONLY assert settings.instructions_dir == "/path/to/instructions" - assert settings.enable_documentation_resource is True + assert settings.enable_documentation is True From 655da80575156e39442ac772688d5b5d46f3cf82 Mon Sep 17 00:00:00 2001 From: Rohit Kumar Date: Tue, 15 Sep 2026 08:09:15 +0530 Subject: [PATCH 7/7] Make documentation guidance HTTP opt-in only --- packages/datacommons-mcp/.env.sample | 4 ---- packages/datacommons-mcp/README.md | 13 +++++++------ .../datacommons-mcp/datacommons_mcp/app.py | 1 - .../datacommons_mcp/data_models/settings.py | 5 ----- .../datacommons_mcp/middleware.py | 7 +++---- packages/datacommons-mcp/tests/server_test.py | 18 ++++-------------- packages/datacommons-mcp/tests/test_app.py | 1 - .../datacommons-mcp/tests/test_settings.py | 3 --- 8 files changed, 14 insertions(+), 38 deletions(-) diff --git a/packages/datacommons-mcp/.env.sample b/packages/datacommons-mcp/.env.sample index 945ef30c..979cff87 100644 --- a/packages/datacommons-mcp/.env.sample +++ b/packages/datacommons-mcp/.env.sample @@ -38,10 +38,6 @@ DC_API_KEY=your_datacommons_api_key_here # - skills/{skill_name}/SKILL.md # DC_INSTRUCTIONS_DIR=/path/to/custom/instructions -# Expose the official Data Commons documentation resource (optional) -# Defaults to false. Set to true for base Data Commons deployments. -# DC_ENABLE_DOCUMENTATION_RESOURCE=true - # ============================================================================= # NON-PROD / LOCAL ROOTS (optional) # ============================================================================= diff --git a/packages/datacommons-mcp/README.md b/packages/datacommons-mcp/README.md index 224b1c5a..b49a7614 100644 --- a/packages/datacommons-mcp/README.md +++ b/packages/datacommons-mcp/README.md @@ -50,10 +50,8 @@ uvx datacommons-mcp serve stdio ### Optional documentation guidance -`DC_ENABLE_DOCUMENTATION` sets the server default (false when unset). -For stdio, pass this environment variable when launching the server process. - -HTTP clients can override the default in their MCP connection configuration: +Documentation guidance is disabled by default. HTTP clients, including clients +connecting to a local HTTP server, can opt in through their MCP connection configuration: ```json "headers": { @@ -61,11 +59,14 @@ HTTP clients can override the default in their MCP connection configuration: } ``` -Use `"false"` to opt out even when the server default is enabled. Omit the header -to inherit the server default. Values are case-insensitive and surrounding +Use `"false"` or omit the header to leave guidance disabled. +Values are case-insensitive and surrounding whitespace is ignored; other values produce an invalid-parameters error. Send the same preference on every request and reconnect after changing it. +There is no server environment setting for this feature. Stdio connections do +not receive the optional documentation guidance; their tools and skills are unchanged. + When enabled, a documentation routing hint follows the existing server instructions and directs the client to fetch https://docs.datacommons.org/llms.txt for relevant documentation questions. The client needs URL-reading capability diff --git a/packages/datacommons-mcp/datacommons_mcp/app.py b/packages/datacommons-mcp/datacommons_mcp/app.py index b79968f5..5f2c0a8c 100644 --- a/packages/datacommons-mcp/datacommons_mcp/app.py +++ b/packages/datacommons-mcp/datacommons_mcp/app.py @@ -92,7 +92,6 @@ async def lifespan(_server: FastMCP) -> AsyncIterator[dict[str, Any]]: ) self.mcp.add_middleware( DocumentationMiddleware( - enabled=self.settings.enable_documentation, base_instructions=base_instructions, documentation_instructions=documentation_instructions, ) diff --git a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py index 6a38bf27..4a655300 100644 --- a/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py +++ b/packages/datacommons-mcp/datacommons_mcp/data_models/settings.py @@ -46,8 +46,3 @@ class DCSettings(BaseSettings): alias="DC_INSTRUCTIONS_DIR", description="Directory containing custom instruction files (markdown overrides)", ) - enable_documentation: bool = Field( - default=False, - alias="DC_ENABLE_DOCUMENTATION", - description="Include guidance for accessing official Data Commons documentation", - ) diff --git a/packages/datacommons-mcp/datacommons_mcp/middleware.py b/packages/datacommons-mcp/datacommons_mcp/middleware.py index 2750569d..e6b5357a 100644 --- a/packages/datacommons-mcp/datacommons_mcp/middleware.py +++ b/packages/datacommons-mcp/datacommons_mcp/middleware.py @@ -18,19 +18,18 @@ class DocumentationMiddleware(Middleware): - """Apply the client's documentation preference, falling back to the server.""" + """Enable documentation guidance only when the HTTP client opts in.""" def __init__( - self, *, enabled: bool, base_instructions: str, documentation_instructions: str + self, *, base_instructions: str, documentation_instructions: str ) -> None: - self._default_enabled = enabled self._base_instructions = base_instructions self._documentation_instructions = documentation_instructions def _enabled(self) -> bool: value = get_http_headers().get(DOCUMENTATION_HEADER.lower()) if value is None: - return self._default_enabled + return False value = value.strip().lower() if value not in ("true", "false"): raise McpError( diff --git a/packages/datacommons-mcp/tests/server_test.py b/packages/datacommons-mcp/tests/server_test.py index e2b16bcf..548196f5 100644 --- a/packages/datacommons-mcp/tests/server_test.py +++ b/packages/datacommons-mcp/tests/server_test.py @@ -34,9 +34,7 @@ def documentation_app(monkeypatch, tmp_path, create_test_file): @pytest.mark.asyncio -@pytest.mark.parametrize("default", [False, True]) -async def test_documentation_http_overrides(default, documentation_app, monkeypatch): - monkeypatch.setenv("DC_ENABLE_DOCUMENTATION", str(default)) +async def test_documentation_http_opt_in(documentation_app, monkeypatch): app = documentation_app() server = app.mcp original = server.instructions @@ -51,7 +49,7 @@ def unrelated(): async with run_server_async(server) as url: async def check(header): - enabled = default if header is None else header.lower() == "true" + enabled = header is not None and header.lower() == "true" headers = {} if header is None else {DOCUMENTATION_HEADER: header} async with Client(StreamableHttpTransport(url, headers=headers)) as client: result = await client.initialize() @@ -82,19 +80,12 @@ async def check(header): @pytest.mark.asyncio -@pytest.mark.parametrize("default", [None, "false", "true"]) -async def test_documentation_without_http_header( - default, documentation_app, monkeypatch -): - """Without HTTP headers, the environment controls the instructions as for stdio.""" - if default is not None: - monkeypatch.setenv("DC_ENABLE_DOCUMENTATION", default) +async def test_documentation_without_http_header(documentation_app): + """Headerless transports receive only the base instructions.""" app = documentation_app() async with Client(app.mcp) as client: result = await client.initialize() expected = "Custom server instructions.\n" - if default == "true": - expected = "Custom server instructions.\n\nCustom documentation hint.\n" assert result.instructions == expected @@ -105,7 +96,6 @@ def test_documentation_header_whitespace(value, expected, monkeypatch): lambda: {DOCUMENTATION_HEADER.lower(): value}, ) middleware = DocumentationMiddleware( - enabled=False, base_instructions="base", documentation_instructions="base with documentation", ) diff --git a/packages/datacommons-mcp/tests/test_app.py b/packages/datacommons-mcp/tests/test_app.py index 07368dcf..398259c3 100644 --- a/packages/datacommons-mcp/tests/test_app.py +++ b/packages/datacommons-mcp/tests/test_app.py @@ -73,7 +73,6 @@ def test_app_prepares_documentation_instructions( custom_dir = tmp_path / "instructions" create_test_file("instructions/server.md", "Custom Server Instructions") mock_settings.return_value.instructions_dir = str(custom_dir) - mock_settings.return_value.enable_documentation = True from datacommons_mcp.app import DCApp diff --git a/packages/datacommons-mcp/tests/test_settings.py b/packages/datacommons-mcp/tests/test_settings.py index fcdbff22..97066c43 100644 --- a/packages/datacommons-mcp/tests/test_settings.py +++ b/packages/datacommons-mcp/tests/test_settings.py @@ -36,7 +36,6 @@ def test_loads_with_minimal_config(self): assert settings.agent_api_root == "https://api.datacommons.org/v2" assert settings.search_scope is None assert settings.instructions_dir is None - assert settings.enable_documentation is False def test_loads_with_env_var_overrides(self): """Tests that environment variables override defaults for DCSettings.""" @@ -45,7 +44,6 @@ def test_loads_with_env_var_overrides(self): "DC_AGENT_API_ROOT": "https://custom-agent-api.datacommons.org/v2", "DC_SEARCH_SCOPE": "custom_only", "DC_INSTRUCTIONS_DIR": "/path/to/instructions", - "DC_ENABLE_DOCUMENTATION": "true", } with patch.dict(os.environ, env_vars): settings = DCSettings() @@ -57,4 +55,3 @@ def test_loads_with_env_var_overrides(self): ) assert settings.search_scope == SearchScope.CUSTOM_ONLY assert settings.instructions_dir == "/path/to/instructions" - assert settings.enable_documentation is True