fix/crawler-proxy-config #29
@@ -54,7 +54,7 @@ class CrawlerConfig(BaseModel):
|
|||||||
|
|
||||||
# Proxy settings
|
# Proxy settings
|
||||||
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
||||||
proxy_required: bool = Field(default=True, description="Whether proxy is mandatory")
|
proxy_required: bool = Field(default=False, description="Whether proxy is mandatory (False for 旁路由 auto-proxy)")
|
||||||
|
|
||||||
# Rate limiting
|
# Rate limiting
|
||||||
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
||||||
|
|||||||
@@ -19,7 +19,8 @@ from playwright.sync_api import Route, Response, sync_playwright
|
|||||||
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
|
from tenacity import Retrying, stop_after_attempt, wait_exponential, RetryError
|
||||||
|
|
||||||
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
|
from kb_workflow.metadata.zhihu_extractor import PlaywrightBrowser
|
||||||
from kb_workflow.pipeline.config import get_settings
|
# NOTE: get_settings is imported lazily in _get_cookie() to avoid circular import
|
||||||
|
# Circular: zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||||
from .models import (
|
from .models import (
|
||||||
AnswerData,
|
AnswerData,
|
||||||
ArticleData,
|
ArticleData,
|
||||||
@@ -129,6 +130,9 @@ class ZhihuCrawler:
|
|||||||
Returns:
|
Returns:
|
||||||
Cookie string from settings
|
Cookie string from settings
|
||||||
"""
|
"""
|
||||||
|
# Lazy import to avoid circular dependency:
|
||||||
|
# zhihu_crawler -> pipeline.config -> pipeline.platforms -> zhihu_crawler
|
||||||
|
from kb_workflow.pipeline.config import get_settings
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
return settings.zhihu_cookie.get_secret_value()
|
return settings.zhihu_cookie.get_secret_value()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user