diff --git a/kb-zhihu-metadata/zhihu_article_crawler.py b/kb-zhihu-metadata/zhihu_article_crawler.py index 8498cf0..b3fb725 100644 --- a/kb-zhihu-metadata/zhihu_article_crawler.py +++ b/kb-zhihu-metadata/zhihu_article_crawler.py @@ -6,6 +6,7 @@ import argparse import json import os +import re import sys import subprocess import shutil @@ -62,16 +63,36 @@ def crawl_zhihu_article(article_url: str, output_dir: Path) -> dict: with open(config_file) as f: config_content = f.read() - # 检查是否已配置指定 ID - if "ZHIHU_SPECIFIED_ID_LIST" not in config_content or "[]" in config_content: - # 需要配置文章 URL - article_id = article_url.split("/")[-1] - new_config = config_content.replace( - 'ZHIHU_SPECIFIED_ID_LIST = []', - f'ZHIHU_SPECIFIED_ID_LIST = ["{article_url}"]' - ) - with open(config_file, "w") as f: - f.write(new_config) + # 强制更新配置中的 ID 列表(每次都更新,确保爬取正确的文章) + article_id = article_url.split("/")[-1] + + # 使用正则表达式替换整个 ZHIHU_SPECIFIED_ID_LIST 部分 + pattern = r'(ZHIHU_SPECIFIED_ID_LIST\s*=\s*\[)([\s\S]*?)(\])' + replacement = f'\\1\n "{article_url}", # article\n\\3' + + new_config = re.sub(pattern, replacement, config_content) + + # 如果正则替换失败(没有匹配到),使用简单替换 + if new_config == config_content: + # 直接替换整个配置项 + if 'ZHIHU_SPECIFIED_ID_LIST' in config_content: + # 找到配置项的范围并替换 + lines = config_content.split('\n') + new_lines = [] + in_list = False + for line in lines: + if 'ZHIHU_SPECIFIED_ID_LIST' in line and '=' in line: + new_lines.append(f'ZHIHU_SPECIFIED_ID_LIST = ["{article_url}", # article') + in_list = True + elif in_list and line.strip().startswith(']'): + new_lines.append(line) + in_list = False + elif not in_list: + new_lines.append(line) + new_config = '\n'.join(new_lines) + + with open(config_file, "w") as f: + f.write(new_config) # 运行 MediaCrawler env = os.environ.copy()