自动同步: 2026-03-19 02:10

2026-03-19 02:10:04 +08:00
parent 4d8f8147b6
commit 264d69ca1e
58 changed files with 1416 additions and 234 deletions
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Test ZhihuCrawler with detailed debugging"""
+
+import sys
+sys.path.insert(0, '/home/starmen1265/work/kb-skills')
+
+from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor
+from kb_workflow.pipeline.config import get_settings
+import time
+
+# Get settings
+settings = get_settings()
+cookie = settings.zhihu_cookie.get_secret_value()
+print(f"Cookie length: {len(cookie)}")
+
+# Test URL
+url = "https://www.zhihu.com/question/2016993095078684011/answer/2017037562368246182"
+
+# Create crawler
+crawler = ZhihuCrawler(headless=True, timeout=60)
+
+print("Starting browser...")
+crawler._start_browser_with_proxy(cookie)
+crawler.interceptor = ZhihuApiInterceptor()
+
+# Set up route interception
+crawler._page.route("**/api/v4/**", crawler.interceptor.handle_route)
+
+print(f"Navigating to: {url}")
+
+# Add event listener to log all requests
+def log_request(request):
+    if "/api/v4/" in request.url:
+        print(f"  API Request: {request.url[:100]}")
+
+crawler._page.on("request", log_request)
+
+# Add event listener to log all responses
+def log_response(response):
+    if "/api/v4/" in response.url:
+        print(f"  API Response: {response.url[:100]} (status: {response.status})")
+
+crawler._page.on("response", log_response)
+
+try:
+    crawler._page.goto(url, wait_until="networkidle", timeout=60000)
+    print(f"Page loaded: {crawler._page.title()}")
+    print(f"Page URL: {crawler._page.url}")
+    
+    print("\nWaiting for API data...")
+    time.sleep(5)
+    
+    print(f"\nCaptured data keys: {list(crawler.interceptor.captured_data.keys())}")
+    
+    if "answer" in crawler.interceptor.captured_data:
+        print("✅ Answer data captured!")
+    else:
+        print("❌ No answer data captured")
+        
+        # Try to get page content
+        content = crawler._page.content()
+        print(f"\nPage content length: {len(content)}")
+        
+        # Check if answer is in the page
+        if "2017037562368246182" in content:
+            print("Answer ID found in page content")
+        else:
+            print("Answer ID NOT found in page content")
+        
+        # Take screenshot
+        screenshot_path = "/tmp/zhihu_debug2.png"
+        crawler._page.screenshot(path=screenshot_path, full_page=True)
+        print(f"Screenshot saved to: {screenshot_path}")
+
+except Exception as e:
+    print(f"Error: {e}")
+    import traceback
+    traceback.print_exc()
+
+finally:
+    crawler._browser.close()
+    crawler._playwright.stop()