81 lines
2.5 KiB
Python
81 lines
2.5 KiB
Python
|
import requests
|
|||
|
|
from bs4 import BeautifulSoup
|
|||
|
|
import time
|
|||
|
|
import os
|
|||
|
|
import argparse
|
|||
|
|
|
|||
|
|
# Semantic Scholar API URL（无需API Key）
|
|||
|
|
SEMANTIC_API = "https://api.semanticscholar.org/graph/v1/paper/DOI:{}?fields=title,abstract"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_dois_from_dblp(dblp_url):
|
|||
|
|
resp = requests.get(dblp_url)
|
|||
|
|
soup = BeautifulSoup(resp.text, "html.parser")
|
|||
|
|
doi_links = []
|
|||
|
|
seen = set()
|
|||
|
|
|
|||
|
|
for article in soup.find_all("li", class_="entry"):
|
|||
|
|
for link in article.find_all("a", href=True):
|
|||
|
|
href = link['href']
|
|||
|
|
if "doi.org" in href and "bsky.app" not in href:
|
|||
|
|
if href.startswith("https://doi.org/"):
|
|||
|
|
doi = href.strip()
|
|||
|
|
if doi not in seen:
|
|||
|
|
seen.add(doi)
|
|||
|
|
doi_links.append(doi)
|
|||
|
|
|
|||
|
|
return doi_links # 保持原顺序
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_metadata_from_semantic(doi):
|
|||
|
|
url = SEMANTIC_API.format(doi.split("doi.org/")[-1])
|
|||
|
|
try:
|
|||
|
|
resp = requests.get(url, timeout=10)
|
|||
|
|
if resp.status_code == 200:
|
|||
|
|
return resp.json()
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"Error fetching {doi}: {e}")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def write_markdown_entry(f, paper):
|
|||
|
|
f.write(f"## {paper.get('title', 'No Title')}\n\n")
|
|||
|
|
abstract = paper.get("abstract", "")
|
|||
|
|
if abstract:
|
|||
|
|
f.write(f"**Abstract**:\n\n{abstract.strip()}\n\n")
|
|||
|
|
else:
|
|||
|
|
f.write(f"_No abstract available._\n\n")
|
|||
|
|
f.write("\n---\n\n")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
parser = argparse.ArgumentParser(
|
|||
|
|
description='Crawl paper abstracts from dblp and Semantic Scholar')
|
|||
|
|
parser.add_argument('dblp_url', help='DBLP conference URL')
|
|||
|
|
parser.add_argument('output_file', help='Output markdown file path')
|
|||
|
|
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
# 确保输出目录存在
|
|||
|
|
os.makedirs(os.path.dirname(args.output_file) if os.path.dirname(
|
|||
|
|
args.output_file) else '.', exist_ok=True)
|
|||
|
|
|
|||
|
|
dois = get_dois_from_dblp(args.dblp_url)
|
|||
|
|
print(f"Found {len(dois)} DOIs.")
|
|||
|
|
|
|||
|
|
with open(args.output_file, "w", encoding="utf-8") as f:
|
|||
|
|
for i, doi in enumerate(dois, 1):
|
|||
|
|
print(f"[{i}/{len(dois)}] Fetching metadata for {doi}")
|
|||
|
|
metadata = get_metadata_from_semantic(doi)
|
|||
|
|
if metadata:
|
|||
|
|
write_markdown_entry(f, metadata)
|
|||
|
|
else:
|
|||
|
|
write_markdown_entry(f, {"title": f"DOI: {doi}"})
|
|||
|
|
time.sleep(0.1) # 避免过快请求被封
|
|||
|
|
|
|||
|
|
print(f"\nDone. Saved to: {args.output_file}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|