Auto backup before evolver run - 2026年 03月 10日 星期二 19:00:50 CST
This commit is contained in:
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Short link resolver for link-parser skill.
|
||||
|
||||
Provides utilities for detecting and resolving short links, particularly
|
||||
Bilibili short links (b23.tv) which may be blocked by anti-scraping measures.
|
||||
|
||||
Supported short link services:
|
||||
- Bilibili: b23.tv
|
||||
- Generic URL shorteners (detection only)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Optional, Dict, List, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
class ShortLinkResolver:
|
||||
"""Short link detection and resolution handler"""
|
||||
|
||||
# Pattern for Bilibili short links
|
||||
BILIBILI_SHORT_PATTERN = re.compile(r'https?://b23\.tv/([a-zA-Z0-9]+)')
|
||||
|
||||
# Pattern for other short link services
|
||||
OTHER_SHORT_PATTERNS = {
|
||||
'bit.ly': re.compile(r'https?://bit\.ly/\w+'),
|
||||
't.cn': re.compile(r'https?://t\.cn/\w+'),
|
||||
'dwz.cn': re.compile(r'https?://dwz\.cn/\w+'),
|
||||
'goo.gl': re.compile(r'https?://goo\.gl/\w+'),
|
||||
'tinyurl.com': re.compile(r'https?://tinyurl\.com/\w+'),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Detect short links in text.
|
||||
|
||||
Args:
|
||||
text: Input text to search
|
||||
|
||||
Returns:
|
||||
List of detected short links with metadata:
|
||||
[
|
||||
{
|
||||
'url': 'https://b23.tv/xxxxx',
|
||||
'platform': 'bilibili',
|
||||
'is_short': True
|
||||
},
|
||||
...
|
||||
]
|
||||
"""
|
||||
detected = []
|
||||
|
||||
# Check for Bilibili short links
|
||||
for match in ShortLinkResolver.BILIBILI_SHORT_PATTERN.finditer(text):
|
||||
detected.append({
|
||||
'url': match.group(0),
|
||||
'code': match.group(1),
|
||||
'platform': 'bilibili',
|
||||
'is_short': True
|
||||
})
|
||||
|
||||
# Check for other short link services
|
||||
for platform, pattern in ShortLinkResolver.OTHER_SHORT_PATTERNS.items():
|
||||
for match in pattern.finditer(text):
|
||||
detected.append({
|
||||
'url': match.group(0),
|
||||
'platform': platform,
|
||||
'is_short': True
|
||||
})
|
||||
|
||||
return detected
|
||||
|
||||
@staticmethod
|
||||
def is_bilibili_short_link(url: str) -> bool:
|
||||
"""
|
||||
Check if URL is a Bilibili short link.
|
||||
|
||||
Args:
|
||||
url: URL to check
|
||||
|
||||
Returns:
|
||||
True if URL is a b23.tv short link
|
||||
"""
|
||||
return bool(ShortLinkResolver.BILIBILI_SHORT_PATTERN.match(url))
|
||||
|
||||
@staticmethod
|
||||
def format_short_link_help(short_link: Optional[str] = None) -> str:
|
||||
"""
|
||||
Format help message for short links.
|
||||
|
||||
Args:
|
||||
short_link: Optional short link URL to mention
|
||||
|
||||
Returns:
|
||||
Help message
|
||||
"""
|
||||
lines = [
|
||||
"Detection: Short link",
|
||||
""
|
||||
]
|
||||
|
||||
if short_link:
|
||||
lines.append(f"Short link: {short_link}")
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"Due to anti-scraping protection, short links cannot be automatically resolved.",
|
||||
"Please provide the full URL or the content ID (e.g., BV号 for Bilibili).",
|
||||
"",
|
||||
"Full URL format: https://www.bilibili.com/video/BVxxxxx"
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
@staticmethod
|
||||
def format_short_link_warning(short_links: List[Dict[str, str]]) -> str:
|
||||
"""
|
||||
Format warning message for multiple short links.
|
||||
|
||||
Args:
|
||||
short_links: List of short link metadata
|
||||
|
||||
Returns:
|
||||
Warning message
|
||||
"""
|
||||
if not short_links:
|
||||
return ""
|
||||
|
||||
lines = [
|
||||
"Short links detected:",
|
||||
""
|
||||
]
|
||||
|
||||
for i, link in enumerate(short_links, 1):
|
||||
platform = link.get('platform', 'unknown')
|
||||
url = link['url']
|
||||
|
||||
if platform == 'bilibili':
|
||||
lines.append(f" [{i}] Bilibili short link: {url}")
|
||||
lines.append(" Full URL or BV ID required")
|
||||
else:
|
||||
lines.append(f" [{i}] {platform}: {url}")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class PlaywrightResolver:
|
||||
"""
|
||||
Browser-based short link resolver using Playwright.
|
||||
|
||||
This is an optional implementation for environments where Playwright
|
||||
is available. It can bypass anti-scraping measures by simulating
|
||||
a real browser.
|
||||
|
||||
Requirements:
|
||||
pip install playwright
|
||||
playwright install chromium
|
||||
"""
|
||||
|
||||
def __init__(self, headless: bool = True, timeout: int = 10000):
|
||||
"""
|
||||
Initialize Playwright resolver.
|
||||
|
||||
Args:
|
||||
headless: Whether to run browser in headless mode
|
||||
timeout: Page navigation timeout in milliseconds
|
||||
"""
|
||||
self.headless = headless
|
||||
self.timeout = timeout
|
||||
self._playwright = None
|
||||
|
||||
def _check_playwright_available(self) -> bool:
|
||||
"""Check if Playwright is installed and available"""
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def resolve(self, short_url: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve short link using Playwright browser automation.
|
||||
|
||||
Args:
|
||||
short_url: Short URL to resolve
|
||||
|
||||
Returns:
|
||||
Final URL after redirects, or None if resolution fails
|
||||
"""
|
||||
if not self._check_playwright_available():
|
||||
raise ImportError(
|
||||
"Playwright not installed. Install with: pip install playwright && playwright install chromium"
|
||||
)
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
try:
|
||||
with sync_playwright() as p:
|
||||
# Launch browser
|
||||
browser = p.chromium.launch(headless=self.headless)
|
||||
context = browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
viewport={'width': 1920, 'height': 1080}
|
||||
)
|
||||
page = context.new_page()
|
||||
|
||||
# Navigate to short URL
|
||||
page.goto(short_url, timeout=self.timeout, wait_until='networkidle')
|
||||
|
||||
# Get final URL
|
||||
final_url = page.url
|
||||
|
||||
# Close browser
|
||||
browser.close()
|
||||
|
||||
return final_url
|
||||
|
||||
except Exception as e:
|
||||
print(f"Playwright resolution failed: {e}")
|
||||
return None
|
||||
|
||||
def resolve_bilibili_short(self, short_url: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve Bilibili short link and extract BV ID.
|
||||
|
||||
Args:
|
||||
short_url: Bilibili short URL (e.g., https://b23.tv/xxxxx)
|
||||
|
||||
Returns:
|
||||
Full Bilibili URL with BV ID, or None if resolution fails
|
||||
"""
|
||||
final_url = self.resolve(short_url)
|
||||
if not final_url:
|
||||
return None
|
||||
|
||||
# Check if final URL contains BV ID
|
||||
if 'bilibili.com/video/' in final_url:
|
||||
return final_url
|
||||
|
||||
# Extract BV ID if present in URL
|
||||
bv_pattern = re.compile(r'(BV[a-zA-Z0-9]{10})')
|
||||
match = bv_pattern.search(final_url)
|
||||
if match:
|
||||
return f"https://www.bilibili.com/video/{match.group(1)}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_resolver() -> ShortLinkResolver:
|
||||
"""
|
||||
Get the default resolver instance.
|
||||
|
||||
Returns:
|
||||
ShortLinkResolver instance
|
||||
"""
|
||||
return ShortLinkResolver()
|
||||
|
||||
|
||||
def detect_short_links(text: str) -> List[Dict[str, str]]:
|
||||
"""
|
||||
Detect short links in text.
|
||||
|
||||
Args:
|
||||
text: Input text to search
|
||||
|
||||
Returns:
|
||||
List of detected short links with metadata
|
||||
"""
|
||||
resolver = ShortLinkResolver()
|
||||
return resolver.detect_short_links(text)
|
||||
|
||||
|
||||
# CLI interface for testing
|
||||
def main():
|
||||
"""CLI for testing the short link resolver."""
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 link_parser_shortlink.py 'text to test'")
|
||||
print("\nExamples:")
|
||||
print(' python3 link_parser_shortlink.py "Check this: https://b23.tv/oLBZZfK"')
|
||||
print(' python3 link_parser_shortlink.py "https://bit.ly/xxxxx"')
|
||||
sys.exit(1)
|
||||
|
||||
text = ' '.join(sys.argv[1:])
|
||||
short_links = detect_short_links(text)
|
||||
|
||||
import json
|
||||
print(json.dumps(short_links, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user