83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Test ZhihuCrawler with detailed debugging"""
|
|
|
|
import sys
|
|
sys.path.insert(0, '/home/starmen1265/work/kb-skills')
|
|
|
|
from kb_workflow.crawler.zhihu_crawler import ZhihuCrawler, ZhihuApiInterceptor
|
|
from kb_workflow.pipeline.config import get_settings
|
|
import time
|
|
|
|
# Get settings
|
|
settings = get_settings()
|
|
cookie = settings.zhihu_cookie.get_secret_value()
|
|
print(f"Cookie length: {len(cookie)}")
|
|
|
|
# Test URL
|
|
url = "https://www.zhihu.com/question/2016993095078684011/answer/2017037562368246182"
|
|
|
|
# Create crawler
|
|
crawler = ZhihuCrawler(headless=True, timeout=60)
|
|
|
|
print("Starting browser...")
|
|
crawler._start_browser_with_proxy(cookie)
|
|
crawler.interceptor = ZhihuApiInterceptor()
|
|
|
|
# Set up route interception
|
|
crawler._page.route("**/api/v4/**", crawler.interceptor.handle_route)
|
|
|
|
print(f"Navigating to: {url}")
|
|
|
|
# Add event listener to log all requests
|
|
def log_request(request):
|
|
if "/api/v4/" in request.url:
|
|
print(f" API Request: {request.url[:100]}")
|
|
|
|
crawler._page.on("request", log_request)
|
|
|
|
# Add event listener to log all responses
|
|
def log_response(response):
|
|
if "/api/v4/" in response.url:
|
|
print(f" API Response: {response.url[:100]} (status: {response.status})")
|
|
|
|
crawler._page.on("response", log_response)
|
|
|
|
try:
|
|
crawler._page.goto(url, wait_until="networkidle", timeout=60000)
|
|
print(f"Page loaded: {crawler._page.title()}")
|
|
print(f"Page URL: {crawler._page.url}")
|
|
|
|
print("\nWaiting for API data...")
|
|
time.sleep(5)
|
|
|
|
print(f"\nCaptured data keys: {list(crawler.interceptor.captured_data.keys())}")
|
|
|
|
if "answer" in crawler.interceptor.captured_data:
|
|
print("✅ Answer data captured!")
|
|
else:
|
|
print("❌ No answer data captured")
|
|
|
|
# Try to get page content
|
|
content = crawler._page.content()
|
|
print(f"\nPage content length: {len(content)}")
|
|
|
|
# Check if answer is in the page
|
|
if "2017037562368246182" in content:
|
|
print("Answer ID found in page content")
|
|
else:
|
|
print("Answer ID NOT found in page content")
|
|
|
|
# Take screenshot
|
|
screenshot_path = "/tmp/zhihu_debug2.png"
|
|
crawler._page.screenshot(path=screenshot_path, full_page=True)
|
|
print(f"Screenshot saved to: {screenshot_path}")
|
|
|
|
except Exception as e:
|
|
print(f"Error: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
finally:
|
|
crawler._browser.close()
|
|
crawler._playwright.stop()
|