diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py index 511fde692..e41b96b0c 100644 --- a/crawl4ai/deep_crawling/bff_strategy.py +++ b/crawl4ai/deep_crawling/bff_strategy.py @@ -83,8 +83,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") + # Single-label hostnames (intranet hosts, localhost, service names) + # are valid targets; do not require a dot in the netloc. except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py index dfb759272..976c04b4f 100644 --- a/crawl4ai/deep_crawling/bfs_strategy.py +++ b/crawl4ai/deep_crawling/bfs_strategy.py @@ -70,8 +70,8 @@ async def can_process_url(self, url: str, depth: int) -> bool: raise ValueError("Missing scheme or netloc") if parsed.scheme not in ("http", "https"): raise ValueError("Invalid scheme") - if "." not in parsed.netloc: - raise ValueError("Invalid domain") + # Single-label hostnames (intranet hosts, localhost, service names) + # are valid targets; do not require a dot in the netloc. except Exception as e: self.logger.warning(f"Invalid URL: {url}, error: {e}") return False diff --git a/tests/deep_crawling/test_can_process_url.py b/tests/deep_crawling/test_can_process_url.py new file mode 100644 index 000000000..d50bb84a8 --- /dev/null +++ b/tests/deep_crawling/test_can_process_url.py @@ -0,0 +1,68 @@ +""" +Test Suite: can_process_url URL validation for deep crawl strategies + +Covers: +1. Single-label (dot-less) hostnames such as intranet hosts, localhost and + Docker service names are accepted for http(s) URLs. +2. Regular hostnames, IP literals and ports keep working. +3. URLs without a scheme/netloc and non-http(s) schemes are still rejected. +4. The filter chain still applies for depth > 0, while depth 0 bypasses it. +""" + +import pytest + +from crawl4ai.deep_crawling import ( + BFSDeepCrawlStrategy, + BestFirstCrawlingStrategy, +) +from crawl4ai.deep_crawling.filters import DomainFilter, FilterChain + + +@pytest.mark.parametrize( + "strategy", + [ + pytest.param(BFSDeepCrawlStrategy(max_depth=1), id="bfs"), + pytest.param(BestFirstCrawlingStrategy(max_depth=1), id="bff"), + ], +) +class TestCanProcessUrl: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "url", + [ + "https://intranet/xyz/", + "http://name/xyz", + "https://localhost:8000/docs", + "http://10.0.0.5:8080/", + "https://example.com/blog/post1", + ], + ) + async def test_accepts_valid_urls(self, strategy, url): + assert await strategy.can_process_url(url, 0) is True + assert await strategy.can_process_url(url, 1) is True + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "url", + [ + "intranet/xyz", # missing scheme + "https:///xyz", # empty netloc + "ftp://example.com/file", # unsupported scheme + ], + ) + async def test_rejects_invalid_urls(self, strategy, url): + assert await strategy.can_process_url(url, 0) is False + assert await strategy.can_process_url(url, 1) is False + + @pytest.mark.asyncio + async def test_filter_chain_still_applied_for_deeper_depths(self, strategy): + strategy.filter_chain = FilterChain( + [DomainFilter(allowed_domains=["intranet"])] + ) + + # Single-label host inside the allowed domain passes. + assert await strategy.can_process_url("https://intranet/xyz", 1) is True + # Host outside the allowed domain is rejected by the filter chain. + assert await strategy.can_process_url("https://other/xyz", 1) is False + # Depth 0 bypasses filtering. + assert await strategy.can_process_url("https://other/xyz", 0) is True