From 123e307522ae0bfe5a6272267aa658d0f73d3905 Mon Sep 17 00:00:00 2001 From: starmen1265 Date: Tue, 17 Mar 2026 22:20:38 +0800 Subject: [PATCH] docs(08-03): complete anti-crawl measures plan - Proxy configuration from HTTP_PROXY/HTTPS_PROXY - Random delay (1-3s) between requests - Exponential backoff retry (3 attempts) - x-zse-96 signature handled by browser All 44 crawler tests passing Requirements: NET-01, NET-02, NET-03 completed Co-Authored-By: Claude Opus 4.6 --- .planning/REQUIREMENTS.md | 12 +- .planning/phases/08-crawler/08-03-SUMMARY.md | 88 +++++++++++++++ tests/test_crawler/test_comment_crawler.py | 110 ++++++++++--------- 3 files changed, 151 insertions(+), 59 deletions(-) create mode 100644 .planning/phases/08-crawler/08-03-SUMMARY.md diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index f236cb7..8430847 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -23,9 +23,9 @@ ### 网络与反爬 -- [ ] **NET-01**: 支持代理 IP 配置 -- [ ] **NET-02**: 请求频率控制，避免触发反爬 -- [ ] **NET-03**: 自动获取请求签名（x-zse-96），无需手动逆向 +- [x] **NET-01**: 支持代理 IP 配置 +- [x] **NET-02**: 请求频率控制，避免触发反爬 +- [x] **NET-03**: 自动获取请求签名（x-zse-96），无需手动逆向 ### Pipeline 整合 @@ -119,9 +119,9 @@ | CONT-02 | Phase 8 | Complete | | CONT-03 | Phase 8 | Complete | | CONT-04 | Phase 8 | Complete | -| NET-01 | Phase 8 | Pending | -| NET-02 | Phase 8 | Pending | -| NET-03 | Phase 8 | Pending | +| NET-01 | Phase 8 | Complete | +| NET-02 | Phase 8 | Complete | +| NET-03 | Phase 8 | Complete | | PIPE-01 | Phase 9 | Pending | | PIPE-02 | Phase 9 | Pending | | PIPE-03 | Phase 9 | Pending | diff --git a/.planning/phases/08-crawler/08-03-SUMMARY.md b/.planning/phases/08-crawler/08-03-SUMMARY.md new file mode 100644 index 0000000..043e886 --- /dev/null +++ b/.planning/phases/08-crawler/08-03-SUMMARY.md @@ -0,0 +1,88 @@ +--- +phase: 08-crawler +plan: 03 +subsystem: crawler +tags: [proxy, rate-limiter, retry, anti-crawl, playwright] + +requires: + - phase: 08-01 + provides: ZhihuCrawler base class +provides: + - CrawlerConfig for proxy/rate limiting configuration + - Random delay (1-3s) between requests + - Exponential backoff retry (3 attempts) + - Automatic x-zse-96 signature handling via browser + +tech-stack: + added: + - tenacity (retry library) + patterns: + - Environment variable proxy configuration + - Exponential backoff retry pattern + - Random delay for rate limiting + +key-files: + created: + - kb_workflow/crawler/config.py + - tests/test_crawler/test_proxy_config.py + - tests/test_crawler/test_rate_limiter.py + - tests/test_crawler/test_api_intercept.py + modified: + - kb_workflow/crawler/zhihu_crawler.py + - kb_workflow/crawler/models.py + - tests/test_crawler/test_comment_crawler.py +key-decisions: + - Mandatory proxy configuration (configurable via proxy_required flag) + - Use tenacity library for retry with exponential backoff + - x-zse-96 signature handled automatically by browser via route.fetch() + - Random delay range: 1-3 seconds (configurable) +requirements-completed: [NET-01, NET-02, NET-03] +duration: 15min +completed: 2026-03-17 +--- + +## Summary + +实现了网络反爬措施：强制代理配置、请求频率控制、有限重试机制，以及利用浏览器自动处理 x-zse-96 签名。 + +### What Was Built + +**CrawlerConfig 类** (`kb_workflow/crawler/config.py`) +- `ProxyConfig` - 代理配置模型 +- 从环境变量 `HTTP_PROXY`/`HTTPS_PROXY` 读取代理设置 +- `proxy_required` 标志控制是否强制代理（默认 `True`） +- 随机延迟配置（`min_delay`, `max_delay`） +- 重试配置（`max_retries`, `retry_min_wait`, `retry_max_wait`） + +**ZhihuCrawler 增强** +- `_start_browser_with_proxy()` - 启动浏览器并应用代理配置 +- `_apply_random_delay()` - 应用随机延迟（1-3 秒） +- `_crawl_with_retry()` - 带指数退避的重试逻辑 +- 浏览器自动处理 x-zse-96 签名（通过 `route.fetch()`） + +### Test Coverage + +**44 个测试全部通过** +- `test_proxy_config.py` - 11 个代理配置测试 +- `test_rate_limiter.py` - 8 个频率控制测试 +- `test_api_intercept.py` - 4 个 API 拦截测试 +- 其他测试文件保持通过 + +### Commits + +1. `626fb73` feat(08-03): create CrawlerConfig for proxy and rate limiting +2. `9e1452d` feat(08-03): integrate proxy, delay, and retry into ZhihuCrawler + +### Key Decisions + +1. **强制代理**: 默认 `proxy_required=True`，未配置代理时抛出 `ValueError` +2. **重试机制**: 使用 `tenacity` 库，最多 3 次重试，间隔指数递增（2s → 10s） +3. **签名处理**: 利用浏览器 `route.fetch()` 自动携带 x-zse-96 签名，无需手动逆向 + +### Requirements Completed + +- **NET-01**: 用户可以通过环境变量配置代理 IP ✓ +- **NET-02**: 不配置代理时，爬虫拒绝运行并提示错误 ✓ +- **NET-03**: 每次请求间有随机延迟（1-3秒），失败重试 3 次 ✓ + +## Self-Check: PASSED diff --git a/tests/test_crawler/test_comment_crawler.py b/tests/test_crawler/test_comment_crawler.py index 51d41ac..629c259 100644 --- a/tests/test_crawler/test_comment_crawler.py +++ b/tests/test_crawler/test_comment_crawler.py @@ -5,6 +5,7 @@ from unittest.mock import Mock, patch, MagicMock from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler from kb_workflow.crawler.models import CommentData, CommentsResult +from kb_workflow.crawler.config import CrawlerConfig class TestCommentCrawler: @@ -13,7 +14,8 @@ class TestCommentCrawler: @pytest.fixture def crawler(self): """Create ZhihuCrawler instance for testing.""" - return ZhihuCrawler(headless=True) + config = CrawlerConfig(proxy_required=False) + return ZhihuCrawler(headless=True, config=config) @pytest.fixture def mock_comment_response(self): @@ -48,12 +50,14 @@ class TestCommentCrawler: def test_crawl_comments_returns_comments_result(self, crawler, mock_comment_response): """crawl_comments returns CommentsResult.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): - with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser: - mock_browser = MockBrowser.return_value + with patch.object(crawler, '_start_browser_with_proxy'): + # Mock _page.request.get mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response - mock_browser.page.request.get.return_value = mock_response + + crawler._page = MagicMock() + crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) @@ -64,28 +68,30 @@ class TestCommentCrawler: def test_comments_are_hot_order(self, crawler, mock_comment_response): """Comments use hot (popular) sorting.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): - with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser: - mock_browser = MockBrowser.return_value + with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response - mock_browser.page.request.get.return_value = mock_response + + crawler._page = MagicMock() + crawler._page.request.get.return_value = mock_response crawler.crawl_comments("answers", 12345) # Verify API URL contains order=hot - call_args = mock_browser.page.request.get.call_args + call_args = crawler._page.request.get.call_args assert "order=hot" in call_args[0][0] def test_comment_has_required_fields(self, crawler, mock_comment_response): """Each comment contains required fields.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): - with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser: - mock_browser = MockBrowser.return_value + with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response - mock_browser.page.request.get.return_value = mock_response + + crawler._page = MagicMock() + crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) @@ -98,12 +104,13 @@ class TestCommentCrawler: def test_no_nested_replies(self, crawler, mock_comment_response): """Do not expand nested replies - only first-level comments.""" with patch.object(crawler, '_get_cookie', return_value="test_cookie"): - with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser: - mock_browser = MockBrowser.return_value + with patch.object(crawler, '_start_browser_with_proxy'): mock_response = MagicMock() mock_response.status = 200 mock_response.json.return_value = mock_comment_response - mock_browser.page.request.get.return_value = mock_response + + crawler._page = MagicMock() + crawler._page.request.get.return_value = mock_response result = crawler.crawl_comments("answers", 12345) @@ -130,49 +137,46 @@ class TestCommentCrawler: "voteup_count": 10, "comment_count": 5 } - mock_comment_response = { - "data": [{"id": 1, "content": "Comment", "author": {"name": "User"}}], - "paging": {"totals": 1} - } with patch.object(crawler, '_get_cookie', return_value="test_cookie"): - with patch('kb_workflow.crawler.zhihu_crawler.PlaywrightBrowser') as MockBrowser: - mock_browser = MockBrowser.return_value - # Setup for answer crawl - mock_browser.page.route = MagicMock() - mock_browser.page.request.get.return_value.status = 200 - mock_browser.page.request.get.return_value.json.return_value = mock_comment_response - mock_browser.goto = MagicMock() - mock_browser.close = MagicMock() - - # Create interceptor that will capture answer data - with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor') as MockInterceptor: - mock_interceptor = MockInterceptor.return_value + with patch.object(crawler, '_start_browser_with_proxy'): + with patch.object(crawler, '_apply_random_delay'): + # Mock ZhihuApiInterceptor + from kb_workflow.crawler.zhihu_crawler import ZhihuApiInterceptor + mock_interceptor = MagicMock(spec=ZhihuApiInterceptor) mock_interceptor.captured_data = {"answer": mock_answer_data} - # Mock crawl_comments to return test data - with patch.object(crawler, 'crawl_comments') as mock_comments: - mock_comments.return_value = CommentsResult( - content_type="answers", - content_id=456, - comments=[CommentData(id=1, content="Test comment")], - total_count=1 - ) + with patch('kb_workflow.crawler.zhihu_crawler.ZhihuApiInterceptor', return_value=mock_interceptor): + # Mock page operations + crawler._page = MagicMock() + crawler._page.goto = MagicMock() + crawler._page.route = MagicMock() + crawler._page.wait_for_timeout = MagicMock() - result = crawler.crawl_answer( - "https://www.zhihu.com/question/123/answer/456", - include_comments=True - ) + # Mock crawl_comments + with patch.object(crawler, 'crawl_comments') as mock_comments: + mock_comments.return_value = CommentsResult( + content_type="answers", + content_id=456, + comments=[CommentData(id=1, content="Test comment")], + total_count=1 + ) - # Verify crawl_comments was called with correct params - mock_comments.assert_called_once_with( - content_type="answers", - content_id=456, - include_comments=True - ) + with patch.object(crawler, '_close_browser'): + result = crawler.crawl_answer( + "https://www.zhihu.com/question/123/answer/456", + include_comments=True + ) - # Verify result includes comments - assert result.success - assert result.comments is not None - assert len(result.comments) == 1 - assert result.comments[0].content == "Test comment" + # Verify crawl_comments was called with correct params + mock_comments.assert_called_once_with( + content_type="answers", + content_id=456, + include_comments=True + ) + + # Verify result includes comments + assert result.success + assert result.comments is not None + assert len(result.comments) == 1 + assert result.comments[0].content == "Test comment"