fix/crawler-proxy-config #29
@@ -54,7 +54,7 @@ class CrawlerConfig(BaseModel):
|
||||
|
||||
# Proxy settings
|
||||
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
||||
proxy_required: bool = Field(default=True, description="Whether proxy is mandatory")
|
||||
proxy_required: bool = Field(default=False, description="Whether proxy is mandatory (False for 旁路由 auto-proxy)")
|
||||
|
||||
# Rate limiting
|
||||
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
||||
|
||||
@@ -19,7 +19,8 @@ from playwright.sync_api import Route, Response, sync_playwright
|
||||
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
|
||||
|
||||
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
|
||||
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from .models import (
|
||||
AnswerData,
|
||||
ArticleData,
|
||||
@@ -129,6 +130,9 @@ class ZhihuCrawler:
|
||||
Returns:
|
||||
Cookie string from settings
|
||||
"""
|
||||
# Lazy import to avoid circular dependency:
|
||||
# zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||
from kb_workflow.pipeline.config import get_settings
|
||||
settings = get_settings()
|
||||
return settings.zhihu_cookie.get_secret_value()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user