This repository has been archived on 2026-02-15. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
2026-02-11 01:10:45 +08:00

638 lines
20 KiB
Python

#!/usr/bin/env python3
"""
Link Parser - Extract and resolve IDs from various platform URLs.
Supports:
- Bilibili (video, article, up主, collection, series)
- YouTube (video, channel, playlist)
- Zhihu (question, answer, user, collection)
- Xiaohongshu (note, user)
- Short links (b23.tv, xhslink.com, etc.)
Usage:
python link-parser.py --url "https://www.bilibili.com/video/BV1xx411c7mD"
python link-parser.py --batch urls.txt --output results.json
"""
import argparse
import json
import re
import sys
from typing import Optional, Dict, Any
from urllib.parse import urlparse
from link_parser_shortlink import ShortLinkResolver
class LinkParser:
"""Main link parser class"""
# Bilibili patterns
BILIBILI_VIDEO_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/video/(BV[a-zA-Z0-9]+)')
BILIBILI_VIDEO_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
BILIBILI_ARTICLE_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/(?:article|read)/([a-zA-Z0-9]+)')
BILIBILI_SPACE_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/(\d+)')
BILIBILI_MOBILE_SPACE_PATTERN = re.compile(r'https?://m\.bilibili\.com/space/(\d+)')
BILIBILI_MOBILE_OPUS_PATTERN = re.compile(r'https?://m\.bilibili\.com/opus/(\d+)')
BILIBILI_FAVLIST_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/favlist\?fid=(\d+)')
BILIBILI_OPUS_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/opus/(\d+)')
BILIBILI_SERIES_PATTERN = re.compile(r'https?://(?:space\.)?bilibili\.com/\d+/lists/(\d+)\?type=series')
BILIBILI_READLIST_PATTERN = re.compile(r'https?://(?:www\.)?bilibili\.com/read/readlist/(rl[a-zA-Z0-9]+)')
# YouTube patterns
YOUTUBE_VIDEO_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11})')
YOUTUBE_CHANNEL_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/channel/([a-zA-Z0-9_-]+)')
YOUTUBE_USER_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/user/([a-zA-Z0-9_-]+)')
YOUTUBE_PLAYLIST_PATTERN = re.compile(r'(?:https?://)?(?:www\.)?youtube\.com/playlist\?list=([a-zA-Z0-9_-]+)')
# Xiaohongshu (小红书) patterns
XIAOHONGSHU_SHORT_PATTERN = re.compile(r'https?://xhslink\.com/o/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/explore/([a-zA-Z0-9]+)')
XIAOHONGSHU_NOTE_PATTERN2 = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/notes/([a-zA-Z0-9]+)')
XIAOHONGSHU_DISCOVERY_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/discovery/item/([a-zA-Z0-9]+)')
XIAOHONGSHU_USER_PATTERN = re.compile(r'https?://(?:www\.)?xiaohongshu\.com/user/profile/(\d+)')
# Zhihu patterns
ZHIHAU_QUESTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/(\d+)')
ZHIHAU_ANSWER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/question/\d+/answer/(\d+)')
ZHIHAU_USER_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/people/([a-zA-Z0-9_-]+)')
ZHIHAU_COLUMN_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/column/([a-zA-Z0-9_-]+)')
ZHIHAU_COLLECTION_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/collection/(\d+)')
# Additional Zhihu patterns
ZHIHAU_ZHUANLAN_PATTERN = re.compile(r'https?://zhuanlan\.zhihu\.com/p/(\d+)')
ZHIHAU_ORG_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/([a-zA-Z0-9_-]+)')
ZHIHAU_ORG_ANSWERS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/answers')
ZHIHAU_ORG_VIDEOS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/zvideos')
ZHIHAU_ORG_ASKS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/asks')
ZHIHAU_ORG_POSTS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/posts')
ZHIHAU_ORG_PINS_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/org/[a-zA-Z0-9_-]+/pins')
ZHIHAU_ZVIDEO_PATTERN = re.compile(r'https?://(?:www\.)?zhihu\.com/zvideo/(\d+)')
def __init__(self):
self.shortlink_resolver = ShortLinkResolver()
def resolve_shortlink(self, url: str) -> Optional[str]:
"""
Resolve short link to get final URL.
Args:
url: Short URL to resolve
Returns:
Final URL after resolution, or None if not resolvable
"""
# Check if it's a known short link pattern
if self.shortlink_resolver.is_bilibili_short_link(url):
# Try to resolve using requests (follows redirects)
try:
import requests
response = requests.head(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
# Fallback: return original URL
return url
# Check if it's Xiaohongshu short link
if self.XIAOHONGSHU_SHORT_PATTERN.match(url):
try:
import requests
response = requests.get(url, allow_redirects=True, timeout=10)
return response.url
except Exception:
return url
return url
def extract_bilibili_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Bilibili ID from URL.
Args:
url: Bilibili URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try series/playlist (must check before space pattern)
match = self.BILIBILI_SERIES_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "playlist",
"id": match.group(1)
}
# Try readlist (专栏合集)
match = self.BILIBILI_READLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "readlist",
"id": match.group(1)
}
# Try favlist (收藏夹)
match = self.BILIBILI_FAVLIST_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "favlist",
"id": match.group(1)
}
# Try opus (动态)
match = self.BILIBILI_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try mobile opus (移动端动态)
match = self.BILIBILI_MOBILE_OPUS_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "opus",
"id": match.group(1)
}
# Try video
match = self.BILIBILI_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "video",
"id": match.group(1)
}
# Try article/read
match = self.BILIBILI_ARTICLE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "article",
"id": match.group(1)
}
# Try space (UP主) - must check after specific patterns
match = self.BILIBILI_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
# Try mobile space (移动端UP主空间)
match = self.BILIBILI_MOBILE_SPACE_PATTERN.match(url)
if match:
return {
"platform": "bilibili",
"type": "up主",
"id": match.group(1)
}
return None
def extract_youtube_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract YouTube ID from URL.
Args:
url: YouTube URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try video
match = self.YOUTUBE_VIDEO_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "video",
"id": match.group(1)
}
# Try channel
match = self.YOUTUBE_CHANNEL_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "channel",
"id": match.group(1)
}
# Try user
match = self.YOUTUBE_USER_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "user",
"id": match.group(1)
}
# Try playlist
match = self.YOUTUBE_PLAYLIST_PATTERN.match(url)
if match:
return {
"platform": "youtube",
"type": "playlist",
"id": match.group(1)
}
return None
def extract_zhihu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Zhihu ID from URL.
Args:
url: Zhihu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try question
match = self.ZHIHAU_QUESTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "question",
"id": match.group(1)
}
# Try answer
match = self.ZHIHAU_ANSWER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "answer",
"id": match.group(1)
}
# Try user
match = self.ZHIHAU_USER_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "user",
"id": match.group(1)
}
# Try column
match = self.ZHIHAU_COLUMN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "column",
"id": match.group(1)
}
# Try collection
match = self.ZHIHAU_COLLECTION_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "collection",
"id": match.group(1)
}
# Try zhuanlan article (专栏文章)
match = self.ZHIHAU_ZHUANLAN_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zhuanlan",
"id": match.group(1)
}
# Try org page (机构页面)
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org",
"id": match.group(1)
}
# Try org answers (机构回答)
if self.ZHIHAU_ORG_ANSWERS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_answers",
"id": match.group(1)
}
# Try org zvideos (机构视频)
if self.ZHIHAU_ORG_VIDEOS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_zvideos",
"id": match.group(1)
}
# Try org asks (机构提问)
if self.ZHIHAU_ORG_ASKS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_asks",
"id": match.group(1)
}
# Try org posts (机构文章)
if self.ZHIHAU_ORG_POSTS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_posts",
"id": match.group(1)
}
# Try org pins (机构想法)
if self.ZHIHAU_ORG_PINS_PATTERN.match(url):
match = self.ZHIHAU_ORG_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "org_pins",
"id": match.group(1)
}
# Try zvideo (知乎视频)
match = self.ZHIHAU_ZVIDEO_PATTERN.match(url)
if match:
return {
"platform": "zhihu",
"type": "zvideo",
"id": match.group(1)
}
return None
def extract_xiaohongshu_id(self, url: str) -> Optional[Dict[str, str]]:
"""
Extract Xiaohongshu ID from URL.
Args:
url: Xiaohongshu URL
Returns:
Dict with platform, type, and id fields, or None if not matched
"""
# Try short link
match = self.XIAOHONGSHU_SHORT_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try explore link
match = self.XIAOHONGSHU_NOTE_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try notes link
match = self.XIAOHONGSHU_NOTE_PATTERN2.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try discovery item link (resolved from short link)
match = self.XIAOHONGSHU_DISCOVERY_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "note",
"id": match.group(1)
}
# Try user profile
match = self.XIAOHONGSHU_USER_PATTERN.match(url)
if match:
return {
"platform": "xiaohongshu",
"type": "user",
"id": match.group(1)
}
return None
def parse(self, url: str) -> Dict[str, Any]:
"""
Parse URL and extract platform-specific ID.
Args:
url: URL to parse
Returns:
Dict with platform, type, id, and original_url fields
"""
result = {
"platform": None,
"type": None,
"id": None,
"original_url": url,
"resolved_url": None,
"success": False,
"error": None
}
# First, resolve short links
resolved = self.resolve_shortlink(url)
if resolved:
result["resolved_url"] = resolved
url = resolved
# Try each platform
bilibili_result = self.extract_bilibili_id(url)
if bilibili_result:
result.update(bilibili_result)
result["success"] = True
return result
youtube_result = self.extract_youtube_id(url)
if youtube_result:
result.update(youtube_result)
result["success"] = True
return result
zhihu_result = self.extract_zhihu_id(url)
if zhihu_result:
result.update(zhihu_result)
result["success"] = True
return result
# Try Xiaohongshu
xiaohongshu_result = self.extract_xiaohongshu_id(url)
if xiaohongshu_result:
result.update(xiaohongshu_result)
result["success"] = True
return result
# Check for short links that need resolution
short_links = self.shortlink_resolver.detect_short_links(url)
if short_links:
result["error"] = "Short link detected. Please provide the full URL."
return result
result["error"] = "Unsupported URL format"
return result
def format_output(data: Dict[str, Any], output_format: str = "json") -> str:
"""Format output based on specified format."""
if output_format == "json":
return json.dumps(data, ensure_ascii=False, indent=2)
elif output_format == "compact":
return json.dumps(data, ensure_ascii=False, separators=(',', ':'))
elif output_format == "text":
lines = []
for key, value in data.items():
if value is not None:
lines.append(f"{key}: {value}")
return "\n".join(lines)
else:
return json.dumps(data, ensure_ascii=False, indent=2)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description="Parse and extract IDs from various platform URLs",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
%(prog)s --url "https://www.bilibili.com/video/BV1xx411c7mD"
%(prog)s --url "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
%(prog)s --url "https://www.zhihu.com/question/266309957"
%(prog)s --batch urls.txt --output results.json
"""
)
parser.add_argument(
"--url", "-u",
help="Single URL to parse"
)
parser.add_argument(
"--batch", "-b",
help="File containing URLs to parse (one per line)"
)
parser.add_argument(
"--output", "-o",
help="Output file path (for batch mode)"
)
parser.add_argument(
"--format", "-f",
choices=["json", "compact", "text"],
default="json",
help="Output format (default: json)"
)
args = parser.parse_args()
# Check if we're in shortlink help mode
if len(sys.argv) == 2 and sys.argv[1] in ['--help', '-h']:
parser.print_help()
return 0
if not args.url and not args.batch:
parser.print_help()
print("\nError: Please provide --url or --batch")
return 1
link_parser = LinkParser()
if args.url:
# Single URL mode
parsed = link_parser.parse(args.url)
# 统一输出格式
if parsed["success"]:
output = {
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None,
"message": "URL解析成功"
}
else:
output = {
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {args.url}"
}
print(json.dumps(output, ensure_ascii=False, indent=2))
return 0 if output["success"] else 1
elif args.batch:
try:
with open(args.batch, 'r', encoding='utf-8') as f:
urls = [line.strip() for line in f if line.strip()]
except Exception as e:
print(json.dumps({
"success": False,
"error": "读取文件失败",
"detail": str(e)
}, ensure_ascii=False, indent=2))
return 1
results = []
for url in urls:
parsed = link_parser.parse(url)
if parsed["success"]:
results.append({
"success": True,
"platform": parsed.get("platform"),
"type": parsed.get("type"),
"id": parsed.get("id"),
"filename": f"{parsed.get('platform')}_{parsed.get('id')}.link.json" if parsed.get("platform") and parsed.get("id") else None
})
else:
results.append({
"success": False,
"error": parsed.get("error", "解析失败"),
"detail": f"URL: {url}"
})
if args.output:
with open(args.output, 'w', encoding='utf-8') as f:
json.dump(results, f, ensure_ascii=False, indent=2)
print(json.dumps({
"success": True,
"message": f"结果已保存到 {args.output}",
"total": len(results)
}, ensure_ascii=False, indent=2))
else:
for result in results:
print(json.dumps(result, ensure_ascii=False, indent=2))
print("-" * 40)
return 0
if __name__ == '__main__':
sys.exit(main())