feat(10-02): integrate ProxyPool into ZhihuCrawler

- Add proxy_pool parameter to ZhihuCrawler constructor
- Add _current_proxy tracking for health check feedback
- Implement _parse_proxy_url static method for URL parsing
- Implement _on_request_success/failure for proxy health feedback
- Prioritize proxy_pool over config.proxy for multi-proxy support
- Maintain backward compatibility with single proxy config

Tests:
- test_crawler_accepts_proxy_pool
- test_crawler_without_proxy_pool (backward compatibility)
- test_parse_proxy_url (HTTP, auth, SOCKS5)
- test_proxy_pool_takes_priority_over_config

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-18 16:48:48 +08:00
parent 36a45095c1
commit b66bcc08b1
2 changed files with 122 additions and 4 deletions
+70 -1
View File
@@ -6,12 +6,14 @@ Tests cover:
- Round-robin proxy rotation
- Empty pool handling
- Single proxy handling
- Health check and cooldown recovery
- ZhihuCrawler integration with ProxyPool
"""
import os
import pytest
from pathlib import Path
from unittest.mock import patch
from unittest.mock import patch, MagicMock
# Import will fail until proxy_pool module is implemented (RED phase)
from kb_workflow.crawler.proxy_pool import ProxyPool, ProxyEntry, ProxyPoolConfig
@@ -304,3 +306,70 @@ class TestProxyPoolHealthCheck:
pool._entries[1].record_failure(max_fail_count=3)
assert pool.get_available_count() == 1
class TestZhihuCrawlerProxyPoolIntegration:
"""Tests for ZhihuCrawler integration with ProxyPool."""
def test_crawler_accepts_proxy_pool(self):
"""Test that ZhihuCrawler accepts proxy_pool parameter."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
config = ProxyPoolConfig(
proxies=["http://proxy-a:8080", "http://proxy-b:8080"]
)
pool = ProxyPool(config)
crawler = ZhihuCrawler(proxy_pool=pool)
assert crawler.proxy_pool is pool
assert crawler.proxy_pool.has_proxies()
def test_crawler_without_proxy_pool(self):
"""Test that ZhihuCrawler works without proxy_pool (backward compatibility)."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
crawler = ZhihuCrawler()
assert crawler.proxy_pool is None
def test_parse_proxy_url(self):
"""Test _parse_proxy_url static method."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
# Test HTTP proxy without auth
result = ZhihuCrawler._parse_proxy_url("http://127.0.0.1:7890")
assert result == {"server": "http://127.0.0.1:7890"}
# Test HTTP proxy with auth
result = ZhihuCrawler._parse_proxy_url("http://user:pass@proxy.example.com:8080")
assert result == {
"server": "http://proxy.example.com:8080",
"username": "user",
"password": "pass"
}
# Test SOCKS5 proxy
result = ZhihuCrawler._parse_proxy_url("socks5://127.0.0.1:1080")
assert result == {"server": "socks5://127.0.0.1:1080"}
def test_proxy_pool_takes_priority_over_config(self):
"""Test that proxy_pool takes priority over config.proxy."""
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
from kb_workflow.crawler.config import CrawlerConfig, ProxyConfig
# Create config with single proxy
single_proxy_config = CrawlerConfig(
proxy=ProxyConfig(server="http://config-proxy:9999")
)
# Create proxy pool with different proxies
pool = ProxyPool(ProxyPoolConfig(
proxies=["http://pool-proxy:8080"]
))
crawler = ZhihuCrawler(config=single_proxy_config, proxy_pool=pool)
# Verify proxy_pool is set
assert crawler.proxy_pool is pool
assert crawler.config.proxy is not None