Compare commits
24 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cb540eb113 | |||
| 9c8ac34138 | |||
| 94a51abed8 | |||
| 9d2433322a | |||
| e1d3218a44 | |||
| d88f1a3b8d | |||
| 5ccabec3e4 | |||
| 289be86fc6 | |||
| 79cf9635c6 | |||
| bc5e02249d | |||
| 36de6cf6dd | |||
| 23254afc49 | |||
| 00b8e103c0 | |||
| 7a1b2fc244 | |||
| 61c4fd401f | |||
| 71176c9a12 | |||
| b5cd81d6e7 | |||
| 46aab1d9f4 | |||
| ed1de1a5b1 | |||
| b89627b8ce | |||
| 33c2fd4bcd | |||
| b3e40b3344 | |||
| 9f12b35e6d | |||
| 4fde411be3 |
@@ -3,6 +3,6 @@
|
|||||||
知识库工作流管道入口。
|
知识库工作流管道入口。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .kb_workflow import process_command, main
|
from .cli import process_command, main
|
||||||
|
|
||||||
__all__ = ["process_command", "main"]
|
__all__ = ["process_command", "main"]
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
KB Workflow 包入口点
|
||||||
|
|
||||||
|
支持通过 python -m kb_workflow 直接运行 CLI。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .cli import main
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -9,11 +9,15 @@ KB Workflow - Knowledge Base Workflow Orchestrator
|
|||||||
内部使用 kb_workflow.pipeline 包实现完整的内容处理流程。
|
内部使用 kb_workflow.pipeline 包实现完整的内容处理流程。
|
||||||
|
|
||||||
使用方式:
|
使用方式:
|
||||||
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
python -m kb_workflow process "https://www.bilibili.com/video/BVxxx"
|
||||||
python kb_workflow.py process "https://..." -o /path/to/output
|
python -m kb_workflow process "https://..." -o /path/to/output
|
||||||
python kb_workflow.py login # QR code login
|
python -m kb_workflow login # QR code login
|
||||||
python kb_workflow.py login --import-cookie cookies.json
|
python -m kb_workflow login --import-cookie cookies.json
|
||||||
python kb_workflow.py login --status
|
python -m kb_workflow login --status
|
||||||
|
|
||||||
|
或者通过包接口调用:
|
||||||
|
from kb_workflow import process_command
|
||||||
|
result = process_command("https://...")
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
@@ -41,13 +45,13 @@ def process_command(
|
|||||||
Dictionary with success status, content_dir, and results
|
Dictionary with success status, content_dir, and results
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
from kb_workflow.pipeline.config import get_settings
|
from .pipeline.config import get_settings
|
||||||
from kb_workflow.pipeline.pipeline import Pipeline
|
from .pipeline.pipeline import Pipeline
|
||||||
from kb_workflow.pipeline.processors import (
|
from .pipeline.processors import (
|
||||||
ASRProcessor, AnalyzeProcessor, ArchiveProcessor,
|
ASRProcessor, AnalyzeProcessor, ArchiveProcessor,
|
||||||
DownloadProcessor, ParseProcessor
|
DownloadProcessor, ParseProcessor
|
||||||
)
|
)
|
||||||
from kb_workflow.pipeline.schemas import PipelineConfig
|
from .pipeline.schemas import PipelineConfig
|
||||||
|
|
||||||
async def run_pipeline():
|
async def run_pipeline():
|
||||||
# Determine output directory - default to intermediate_path
|
# Determine output directory - default to intermediate_path
|
||||||
@@ -119,9 +123,9 @@ def login_command(
|
|||||||
Dictionary with login result
|
Dictionary with login result
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
from kb_workflow.auth import ZhihuLogin
|
from .auth import ZhihuLogin
|
||||||
from kb_workflow.auth.cookie_manager import CookieManager, parse_json_cookies
|
from .auth.cookie_manager import CookieManager, parse_json_cookies
|
||||||
from kb_workflow.pipeline.config import get_settings
|
from .pipeline.config import get_settings
|
||||||
|
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
manager = CookieManager()
|
manager = CookieManager()
|
||||||
@@ -175,22 +179,22 @@ def main():
|
|||||||
epilog='''
|
epilog='''
|
||||||
使用示例:
|
使用示例:
|
||||||
# 处理URL (通过完整管道)
|
# 处理URL (通过完整管道)
|
||||||
python kb_workflow.py process "https://www.bilibili.com/video/BVxxx"
|
python -m kb_workflow process "https://www.bilibili.com/video/BVxxx"
|
||||||
|
|
||||||
# 指定输出目录
|
# 指定输出目录
|
||||||
python kb_workflow.py process "https://..." -o /path/to/output
|
python -m kb_workflow process "https://..." -o /path/to/output
|
||||||
|
|
||||||
# 强制重新执行(忽略已完成状态)
|
# 强制重新执行(忽略已完成状态)
|
||||||
python kb_workflow.py process "https://..." --no-skip
|
python -m kb_workflow process "https://..." --no-skip
|
||||||
|
|
||||||
# 知乎登录 (二维码扫码)
|
# 知乎登录 (二维码扫码)
|
||||||
python kb_workflow.py login
|
python -m kb_workflow login
|
||||||
|
|
||||||
# 导入 Cookie
|
# 导入 Cookie
|
||||||
python kb_workflow.py login --import-cookie cookies.json
|
python -m kb_workflow login --import-cookie cookies.json
|
||||||
|
|
||||||
# 检查 Cookie 状态
|
# 检查 Cookie 状态
|
||||||
python kb_workflow.py login --status
|
python -m kb_workflow login --status
|
||||||
'''
|
'''
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -54,7 +54,7 @@ class CrawlerConfig(BaseModel):
|
|||||||
|
|
||||||
# Proxy settings
|
# Proxy settings
|
||||||
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
proxy: Optional[ProxyConfig] = Field(None, description="Proxy configuration")
|
||||||
proxy_required: bool = Field(default=True, description="Whether proxy is mandatory")
|
proxy_required: bool = Field(default=False, description="Whether proxy is mandatory (False for 旁路由 auto-proxy)")
|
||||||
|
|
||||||
# Rate limiting
|
# Rate limiting
|
||||||
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
min_delay: float = Field(default=1.0, description="Minimum delay between requests (seconds)")
|
||||||
|
|||||||
Reference in New Issue
Block a user