feat(10-02): integrate ProxyPool into ZhihuCrawler
- Add proxy_pool parameter to ZhihuCrawler constructor - Add _current_proxy tracking for health check feedback - Implement _parse_proxy_url static method for URL parsing - Implement _on_request_success/failure for proxy health feedback - Prioritize proxy_pool over config.proxy for multi-proxy support - Maintain backward compatibility with single proxy config Tests: - test_crawler_accepts_proxy_pool - test_crawler_without_proxy_pool (backward compatibility) - test_parse_proxy_url (HTTP, auth, SOCKS5) - test_proxy_pool_takes_priority_over_config Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -6,12 +6,14 @@ Tests cover:
|
||||
- Round-robin proxy rotation
|
||||
- Empty pool handling
|
||||
- Single proxy handling
|
||||
- Health check and cooldown recovery
|
||||
- ZhihuCrawler integration with ProxyPool
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
# Import will fail until proxy_pool module is implemented (RED phase)
|
||||
from kb_workflow.crawler.proxy_pool import ProxyPool, ProxyEntry, ProxyPoolConfig
|
||||
@@ -304,3 +306,70 @@ class TestProxyPoolHealthCheck:
|
||||
pool._entries[1].record_failure(max_fail_count=3)
|
||||
|
||||
assert pool.get_available_count() == 1
|
||||
|
||||
|
||||
class TestZhihuCrawlerProxyPoolIntegration:
|
||||
"""Tests for ZhihuCrawler integration with ProxyPool."""
|
||||
|
||||
def test_crawler_accepts_proxy_pool(self):
|
||||
"""Test that ZhihuCrawler accepts proxy_pool parameter."""
|
||||
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
||||
|
||||
config = ProxyPoolConfig(
|
||||
proxies=["http://proxy-a:8080", "http://proxy-b:8080"]
|
||||
)
|
||||
pool = ProxyPool(config)
|
||||
|
||||
crawler = ZhihuCrawler(proxy_pool=pool)
|
||||
|
||||
assert crawler.proxy_pool is pool
|
||||
assert crawler.proxy_pool.has_proxies()
|
||||
|
||||
def test_crawler_without_proxy_pool(self):
|
||||
"""Test that ZhihuCrawler works without proxy_pool (backward compatibility)."""
|
||||
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
||||
|
||||
crawler = ZhihuCrawler()
|
||||
|
||||
assert crawler.proxy_pool is None
|
||||
|
||||
def test_parse_proxy_url(self):
|
||||
"""Test _parse_proxy_url static method."""
|
||||
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
||||
|
||||
# Test HTTP proxy without auth
|
||||
result = ZhihuCrawler._parse_proxy_url("http://127.0.0.1:7890")
|
||||
assert result == {"server": "http://127.0.0.1:7890"}
|
||||
|
||||
# Test HTTP proxy with auth
|
||||
result = ZhihuCrawler._parse_proxy_url("http://user:pass@proxy.example.com:8080")
|
||||
assert result == {
|
||||
"server": "http://proxy.example.com:8080",
|
||||
"username": "user",
|
||||
"password": "pass"
|
||||
}
|
||||
|
||||
# Test SOCKS5 proxy
|
||||
result = ZhihuCrawler._parse_proxy_url("socks5://127.0.0.1:1080")
|
||||
assert result == {"server": "socks5://127.0.0.1:1080"}
|
||||
|
||||
def test_proxy_pool_takes_priority_over_config(self):
|
||||
"""Test that proxy_pool takes priority over config.proxy."""
|
||||
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
||||
from kb_workflow.crawler.config import CrawlerConfig, ProxyConfig
|
||||
|
||||
# Create config with single proxy
|
||||
single_proxy_config = CrawlerConfig(
|
||||
proxy=ProxyConfig(server="http://config-proxy:9999")
|
||||
)
|
||||
|
||||
# Create proxy pool with different proxies
|
||||
pool = ProxyPool(ProxyPoolConfig(
|
||||
proxies=["http://pool-proxy:8080"]
|
||||
))
|
||||
|
||||
crawler = ZhihuCrawler(config=single_proxy_config, proxy_pool=pool)
|
||||
|
||||
# Verify proxy_pool is set
|
||||
assert crawler.proxy_pool is pool
|
||||
assert crawler.config.proxy is not None
|
||||
|
||||
Reference in New Issue
Block a user