64 lines
2.0 KiB
Python
64 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Test Zhihu Crawler to debug the issue"""
|
|
|
|
import sys
|
|
sys.path.insert(0, '/home/starmen1265/work/kb-skills')
|
|
|
|
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler
|
|
from kb_workflow.pipeline.config import get_settings
|
|
import time
|
|
|
|
# Get settings
|
|
settings = get_settings()
|
|
cookie = settings.zhihu_cookie.get_secret_value()
|
|
print(f"Cookie length: {len(cookie)}")
|
|
print(f"Cookie starts with: {cookie[:50]}...")
|
|
print()
|
|
|
|
# Test URL
|
|
url = "https://www.zhihu.com/question/2016993095078684011/answer/2017037562368246182"
|
|
|
|
# Create crawler with headless=True (no X server available)
|
|
crawler = ZhihuCrawler(headless=True, timeout=60)
|
|
|
|
print("Starting browser...")
|
|
crawler._start_browser_with_proxy(cookie)
|
|
crawler.interceptor = crawler.__class__.__module__
|
|
from kb_workflow.crawler.zhihu_crawler import ZhihuApiInterceptor
|
|
crawler.interceptor = ZhihuApiInterceptor()
|
|
|
|
# Set up route interception
|
|
crawler._page.route("**/api/v4/**", crawler.interceptor.handle_route)
|
|
|
|
print(f"Navigating to: {url}")
|
|
crawler._page.goto(url, wait_until="networkidle", timeout=60000)
|
|
|
|
print("Waiting for data capture...")
|
|
time.sleep(5)
|
|
|
|
print(f"\nCaptured data keys: {list(crawler.interceptor.captured_data.keys())}")
|
|
|
|
if "answer" in crawler.interceptor.captured_data:
|
|
print("✅ Answer data captured!")
|
|
answer_data = crawler.interceptor.captured_data["answer"]
|
|
if isinstance(answer_data, dict):
|
|
print(f"Answer data type: answer")
|
|
print(f"Keys: {list(answer_data.keys())[:10]}")
|
|
else:
|
|
print("❌ No answer data captured")
|
|
print("\nChecking page content...")
|
|
title = crawler._page.title()
|
|
print(f"Page title: {title}")
|
|
|
|
# Check if we're on login page
|
|
if "登录" in title or "signin" in crawler._page.url:
|
|
print("⚠️ Redirected to login page - Cookie may be expired")
|
|
|
|
# Take screenshot
|
|
screenshot_path = "/tmp/zhihu_debug.png"
|
|
crawler._page.screenshot(path=screenshot_path)
|
|
print(f"Screenshot saved to: {screenshot_path}")
|
|
|
|
crawler._browser.close()
|
|
crawler._playwright.stop()
|