fix/crawler-proxy-config #29

Merged
starmen1265 merged 3 commits from fix/crawler-proxy-config into main 2026-03-18 00:05:49 +08:00
2 changed files with 6 additions and 2 deletions
+1 -1
View File
@@ -54,7 +54,7 @@ class CrawlerConfig(BaseModel):
# Proxy settings
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
proxy_required: bool = Field(default=True, description="Whether proxy is mandatory")
proxy_required: bool = Field(default=False, description="Whether proxy is mandatory (False for 旁路由 auto-proxy)")
# Rate limiting
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
+5 -1
View File
@@ -19,7 +19,8 @@ from playwright.sync_api import Route, Response, sync_playwright
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
from kb_workflow.pipeline.config import get_settings
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
from .models import (
AnswerData,
ArticleData,
@@ -129,6 +130,9 @@ class ZhihuCrawler:
Returns:
Cookie string from settings
"""
# Lazy import to avoid circular dependency:
# zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
from kb_workflow.pipeline.config import get_settings
settings = get_settings()
return settings.zhihu_cookie.get_secret_value()