自动同步: 2026-03-19 02:10

agents/knowledge-base/workspace/test_zhihu_crawler.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Test Zhihu Crawler to debug the issue"""
+
+import sys
+sys.path.insert(0, '/home/starmen1265/work/kb-skills')
+
+from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
+from kb_workflow.pipeline.config import get_settings
+import time
+
+# Get settings
+settings = get_settings()
+cookie = settings.zhihu_cookie.get_secret_value()
+print(f"Cookie length: {len(cookie)}")
+print(f"Cookie starts with: {cookie[:50]}...")
+print()
+
+# Test URL
+url = "https://www.zhihu.com/question/2016993095078684011/answer/2017037562368246182"
+
+# Create crawler with headless=True (no X server available)
+crawler = ZhihuCrawler(headless=True, timeout=60)
+
+print("Starting browser...")
+crawler._start_browser_with_proxy(cookie)
+crawler.interceptor = crawler.__class__.__module__
+from kb_workflow.crawler.zhihu_crawler import ZhihuApiInterceptor
+crawler.interceptor = ZhihuApiInterceptor()
+
+# Set up route interception
+crawler._page.route("**/api/v4/**", crawler.interceptor.handle_route)
+
+print(f"Navigating to: {url}")
+crawler._page.goto(url, wait_until="networkidle", timeout=60000)
+
+print("Waiting for data capture...")
+time.sleep(5)
+
+print(f"\nCaptured data keys: {list(crawler.interceptor.captured_data.keys())}")
+
+if "answer" in crawler.interceptor.captured_data:
+    print("✅ Answer data captured!")
+    answer_data = crawler.interceptor.captured_data["answer"]
+    if isinstance(answer_data, dict):
+        print(f"Answer data type: answer")
+        print(f"Keys: {list(answer_data.keys())[:10]}")
+else:
+    print("❌ No answer data captured")
+    print("\nChecking page content...")
+    title = crawler._page.title()
+    print(f"Page title: {title}")
+    
+    # Check if we're on login page
+    if "登录" in title or "signin" in crawler._page.url:
+        print("⚠️  Redirected to login page - Cookie may be expired")
+    
+    # Take screenshot
+    screenshot_path = "/tmp/zhihu_debug.png"
+    crawler._page.screenshot(path=screenshot_path)
+    print(f"Screenshot saved to: {screenshot_path}")
+
+crawler._browser.close()
+crawler._playwright.stop()