diff --git a/src/research_signal_context_pipelines/research_context_adapter.py b/src/research_signal_context_pipelines/research_context_adapter.py index 7fe2b17..8c2988b 100644 --- a/src/research_signal_context_pipelines/research_context_adapter.py +++ b/src/research_signal_context_pipelines/research_context_adapter.py @@ -10,7 +10,7 @@ from typing import Any from urllib.error import URLError from urllib.parse import urlsplit -from urllib.request import Request, urlopen +from urllib.request import HTTPRedirectHandler, Request, build_opener from xml.etree import ElementTree as ET @@ -19,6 +19,15 @@ WEB_RESEARCH_USER_AGENT = "Mozilla/5.0" +class _NoRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # Only the configured URL was admitted; no follow-up request is allowed. + return None + + +urlopen = build_opener(_NoRedirect()).open + + @dataclass(frozen=True) class ResearchSourceConfig: url: str @@ -294,7 +303,7 @@ def build_context(self, *, pit_timestamp: dt.datetime | None = None) -> dict[str fetched_time = dt.datetime.now(dt.timezone.utc) fetched_at = _isoformat_utc(fetched_time) except (OSError, URLError, TimeoutError, ValueError) as exc: - warnings.append(f"failed to fetch research source {source.url}: {type(exc).__name__}: {exc}") + warnings.append(f"failed to fetch research source: {type(exc).__name__}") continue if cutoff is not None and fetched_time > cutoff: diff --git a/tests/test_research_context_adapter.py b/tests/test_research_context_adapter.py index 4c72a77..cfdda66 100644 --- a/tests/test_research_context_adapter.py +++ b/tests/test_research_context_adapter.py @@ -4,6 +4,10 @@ import json import importlib.util import sys +import io +from email.message import Message +import urllib.request +from urllib.response import addinfourl import pytest from pathlib import Path @@ -12,6 +16,37 @@ from research_signal_context_pipelines import research_context_adapter as adapter_module +@pytest.mark.parametrize("status", [301, 302, 303, 307, 308]) +@pytest.mark.parametrize("target", [ + "https://outside.example/private", "https://synthetic.example/another-path", + "http://127.0.0.1/private", "http://synthetic.example/downgrade", +]) +def test_redirect_is_stopped_before_any_target_request(tmp_path, monkeypatch, status, target): + calls = [] + + def https_open(_handler, request): + calls.append(request.full_url) + headers = Message() + headers["Content-Type"] = "application/rss+xml" + if len(calls) == 1: + headers["Location"] = target + response = addinfourl( + io.BytesIO(_rss("2026-09-04T00:00:00Z").encode()), headers, + request.full_url, status if len(calls) == 1 else 200, + ) + response.msg = "synthetic response" + return response + + monkeypatch.setattr(urllib.request.HTTPSHandler, "https_open", https_open) + monkeypatch.setattr(urllib.request.HTTPHandler, "http_open", https_open) + monkeypatch.setattr(urllib.request, "_opener", None) + context = ResearchContextAdapter(_config(tmp_path)).build_context() + assert calls == ["https://synthetic.example/source"] + assert context["source_count"] == 0 + assert context["research_sources"] == [] + assert any("failed to fetch" in warning for warning in context["warnings"]) + + @pytest.fixture(autouse=True) def clock(monkeypatch): real_datetime = dt.datetime