#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
lit_fetch.py — 多通道合法全文获取（宝锐文献基础设施 · 杠杆2）

把"已知 DOI → 拿全文"这件事固化成一条命令，六通道顺序探测：
  OpenAlex → Semantic Scholar → Unpaywall → Europe PMC(XML) → PMC(idconv→正文) → arXiv(预印本)

子命令：
  get <DOI> [...]                     逐篇探测并下载（落盘 知识沉淀/_全文/OA/*.txt + 结果 JSON）
  list <file.txt>                     从清单文件批量跑（每行一个 DOI）
  report                              汇总最近一次结果 JSON

合法边界：只走开放获取 / 出版社免费 / 预印本 / 仓储副本（绿OA）；不碰盗版镜像。
用法：
  /usr/bin/python3 脚本/lit_fetch.py list /tmp/closed_missing.txt
"""
import json
import os
import re
import sys
import time
import urllib.parse
import urllib.request

HOME = os.path.expanduser('~')
BASE = os.path.join(HOME, 'Desktop/Hermes输出-工作类')
OA_DIR = os.path.join(BASE, '知识沉淀', '_全文', 'OA')
OUT_JSON = os.path.join(BASE, '知识沉淀', '_全文', 'fetch_last_run.json')
KDIR = os.path.join(BASE, '知识沉淀')
AB_DIR = os.path.join(BASE, '知识沉淀', '_全文', '_摘要级')
PROBE_JSON = os.path.join(BASE, '知识沉淀', '_全文', 'probe_last_run.json')
MAIL = 'research@biorisales.site'
UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36'

try:
    import fitz  # pymupdf
except Exception:
    fitz = None


def norm_id(s):
    return re.sub(r'[^a-z0-9]', '', str(s).lower())


def fname_for(doi):
    return doi.replace('/', '_').replace(':', '-') + '.txt'


class FetchError(Exception):
    pass


def http(url, timeout=25, accept=None, binary=False):
    req = urllib.request.Request(url, headers={'User-Agent': UA,
                                               'Accept': accept or '*/*'})
    last = ''
    for attempt in (1, 2):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as r:
                data = r.read()
            return data if binary else data.decode('utf-8', 'ignore')
        except Exception as e:
            last = '%s: %s' % (type(e).__name__, str(e)[:100])
            if attempt == 1:
                time.sleep(1.5)
    raise FetchError(last)


def jget(url, timeout=25):
    try:
        return json.loads(http(url, timeout=timeout, accept='application/json'))
    except Exception as e:
        return {'_error': str(e)}


def clean_html(t):
    t = re.sub(r'<(script|style|nav|header|footer)\b.*?</\1>', ' ', t, flags=re.S | re.I)
    t = re.sub(r'<[^>]+>', ' ', t)
    import html as H
    t = H.unescape(t)
    return re.sub(r'\s+', ' ', t).strip()


# ---------------- 通道 ----------------
def ch_openalex(doi):
    d = jget('https://api.openalex.org/works/doi:%s?mailto=%s' % (urllib.parse.quote(doi), MAIL))
    if not d or d.get('_error'):
        return {}
    locs = [d.get('best_oa_location')] + list(d.get('locations') or [])
    pdfs, landing = [], []
    for l in locs:
        if not l:
            continue
        if l.get('pdf_url'):
            pdfs.append(l['pdf_url'])
        if l.get('landing_page_url'):
            landing.append(l['landing_page_url'])
    return {'title': (d.get('title') or '')[:200], 'pdfs': pdfs, 'landing': landing,
            'oa': bool((d.get('open_access') or {}).get('is_oa')),
            'venue': ((d.get('primary_location') or {}).get('source') or {}).get('display_name', '')}


def ch_s2(doi):
    d = jget('https://api.semanticscholar.org/graph/v1/paper/DOI:%s?fields=title,isOpenAccess,openAccessPdf,externalIds' % urllib.parse.quote(doi))
    if not d or d.get('_error'):
        return {}
    pdf = (d.get('openAccessPdf') or {}).get('url')
    ext = d.get('externalIds') or {}
    return {'title': (d.get('title') or '')[:200], 'pdfs': [pdf] if pdf else [],
            'arxiv': ext.get('ArXiv'), 'pmcid': ext.get('PubMedCentral')}


def ch_unpaywall(doi):
    d = jget('https://api.unpaywall.org/v2/%s?email=%s' % (urllib.parse.quote(doi), MAIL))
    if not d or d.get('_error') or not d.get('is_oa'):
        return {}
    out = {'pdfs': [], 'landing': []}
    for l in ([d.get('best_oa_location')] + list(d.get('oa_locations') or [])):
        if not l:
            continue
        if l.get('url_for_pdf'):
            out['pdfs'].append(l['url_for_pdf'])
        if l.get('url'):
            out['landing'].append(l['url'])
    return out


def ch_epmc(doi):
    q = urllib.parse.quote('DOI:"%s"' % doi)
    d = jget('https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=%s&resultType=core&format=json&pageSize=1' % q)
    try:
        res = (d.get('resultList') or {}).get('result') or []
    except Exception:
        return {}
    if not res:
        return {}
    r = res[0]
    return {'title': (r.get('title') or '')[:200], 'pmcid': r.get('pmcid'),
            'inEPMC': r.get('inEPMC'), 'isOpenAccess': r.get('isOpenAccess')}


def ch_pmcidconv(doi):
    d = jget('https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json&tool=baorui-lit&email=%s'
             % (urllib.parse.quote(doi), MAIL))
    try:
        recs = d.get('records') or []
    except Exception:
        return {}
    if not recs:
        return {}
    return {'pmcid': recs[0].get('pmcid'), 'pmid': recs[0].get('pmid')}


def ch_arxiv(title):
    if not title:
        return {}
    words = re.sub(r'[^A-Za-z0-9 ]', ' ', title).split()
    q = ' AND '.join('ti:"%s"' % w for w in words[:6])
    try:
        xml = http('http://export.arxiv.org/api/query?search_query=%s&max_results=3' % urllib.parse.quote(q))
    except FetchError:
        return {}
    ids = re.findall(r'<id>http://arxiv.org/abs/([^<]+)</id>', xml)
    return {'arxiv_ids': ids}


def ch_crossref(doi):
    d = jget('https://api.crossref.org/works/%s' % urllib.parse.quote(doi))
    if not d or d.get('_error'):
        return {}
    m = d.get('message') or {}
    return {'title': ((m.get('title') or [''])[0])[:200],
            'publisher': m.get('publisher', ''),
            'journal': ((m.get('container-title') or [''])[0])[:80],
            'year': ((m.get('issued') or {}).get('date-parts') or [[None]])[0][0]}


# ---------------- 下载 ----------------
def download(url, doi, kind='pdf'):
    """kind: pdf|html|xml"""
    try:
        raw = http(url, timeout=40, binary=(kind == 'pdf'))
    except Exception as e:
        return None, 'fetch失败: %s' % str(e)[:80]
    if kind == 'pdf':
        if not raw[:5].startswith(b'%PDF'):
            return None, '非 PDF 内容'
        if fitz is None:
            return None, 'pymupdf 未安装'
        try:
            doc = fitz.open(stream=raw, filetype='pdf')
            txt = '\n\n'.join(p.get_text() for p in doc)
        except Exception as e:
            return None, 'PDF 解析失败: %s' % str(e)[:60]
    elif kind == 'xml':
        txt = raw
    else:
        txt = clean_html(raw)
    txt = txt.strip()
    if len(txt) < 2000:
        return None, '正文过短(%d 字符)' % len(txt)
    if 'PMC' in url and 'access denied' in txt[:5000].lower():
        return None, '访问被拒'
    return txt, 'ok(len=%d)' % len(txt)


def try_channels(doi):
    """顺序探测，返回 (状态, 通道, 说明, 正文文本)"""
    log = []
    meta = {}
    # 1. OpenAlex
    oa = ch_openalex(doi)
    if oa:
        meta.update({k: v for k, v in oa.items() if k in ('title', 'venue')})
        log.append('OpenAlex:oa=%s,pdf=%d' % (oa.get('oa'), len(oa.get('pdfs') or [])))
        for u in (oa.get('pdfs') or [])[:3]:
            t, msg = download(u, doi, 'pdf')
            if t:
                return 'OK', 'OpenAlex', u, t, meta, log
            log.append('  OpenAlex-PDF→%s' % msg)
    time.sleep(0.3)
    # 2. Semantic Scholar
    s2 = ch_s2(doi)
    if s2:
        if s2.get('title') and not meta.get('title'):
            meta['title'] = s2['title']
        if s2.get('arxiv'):
            meta['arxiv'] = s2['arxiv']
        log.append('S2:pdf=%d,arxiv=%s' % (len(s2.get('pdfs') or []), s2.get('arxiv')))
        for u in (s2.get('pdfs') or [])[:2]:
            t, msg = download(u, doi, 'pdf')
            if t:
                return 'OK', 'SemanticScholar', u, t, meta, log
            log.append('  S2-PDF→%s' % msg)
    time.sleep(0.4)
    # 3. Unpaywall
    up = ch_unpaywall(doi)
    for u in (up.get('pdfs') or [])[:3]:
        t, msg = download(u, doi, 'pdf')
        if t:
            return 'OK', 'Unpaywall', u, t, meta, log
        log.append('  Unpaywall-PDF→%s' % msg)
    # 4. Europe PMC 全文 XML
    ep = ch_epmc(doi)
    if ep:
        if ep.get('title') and not meta.get('title'):
            meta['title'] = ep['title']
        if ep.get('pmcid'):
            meta['pmcid'] = ep['pmcid']
        log.append('EPMC:pmcid=%s,inEPMC=%s,OA=%s' % (ep.get('pmcid'), ep.get('inEPMC'), ep.get('isOpenAccess')))
        if ep.get('pmcid'):
            try:
                xml = http('https://www.ebi.ac.uk/europepmc/webservices/rest/%s/fullTextXML' % ep['pmcid'])
            except FetchError as e:
                xml = ''
                log.append('  EPMC-XML→%s' % e)
            if xml:
                body = re.sub(r'<[^>]+>', ' ', re.sub(r'<(ref-list|back)\b.*?</\1>', ' ', xml, flags=re.S))
                body = re.sub(r'\s+', ' ', body).strip()
                if len(body) > 3000:
                    return 'OK', 'EuropePMC-XML', ep['pmcid'], body, meta, log
                log.append('  EPMC-XML→正文过短')
    time.sleep(0.3)
    # 5. PMC idconv → PMC 正文 HTML
    pc = ch_pmcidconv(doi)
    pmcid = (pc or {}).get('pmcid') or meta.get('pmcid')
    if pmcid:
        meta['pmcid'] = pmcid
        log.append('idconv:pmcid=%s' % pmcid)
        for u in ('https://pmc.ncbi.nlm.nih.gov/articles/%s/' % pmcid,
                  'https://pmc.ncbi.nlm.nih.gov/articles/%s/pdf/' % pmcid):
            t, msg = download(u, doi, 'pdf' if u.endswith('pdf/') else 'html')
            if t:
                return 'OK', 'PMC-HTML', u, t, meta, log
            log.append('  PMC→%s' % msg)
    time.sleep(0.3)
    # 6. arXiv 预印本
    arx = ch_arxiv(meta.get('title') or '')
    ids = [i for i in (arx.get('arxiv_ids') or []) if not i.startswith('http')]
    if ids:
        log.append('arXiv: %s' % ','.join(ids[:2]))
        for aid in ids[:2]:
            t, msg = download('https://arxiv.org/pdf/%s' % aid, doi, 'pdf')
            if t:
                meta['arxiv'] = aid
                return 'OK', 'arXiv(预印本)', aid, t, meta, log
            log.append('  arXiv-PDF→%s' % msg)
    return 'CLOSED', '-', '-', '', meta, log



# ============================================================================
# 第二级：probe —— 候选 URL 全展开 + HTML 正文兜底 + 标题级 PMC/预印本反查
# 专门对付"标记 OA 但 PDF 403""Unpaywall 说封闭但实际有预印本"两类残篇
# ============================================================================


SUBSCRIPTION_MARKS = (
    'preview of subscription content', 'access options', 'access through your institution',
    'access this article', 'get access to this article', 'purchase pdf', 'buy article',
    'rent this article', 'sign in to continue', 'institutional access', 'subscribe to journal',
    'this article is not open access', 'log in to your account', 'choose your access',
    'we are sorry, but the page you', 'your institution is not subscribed',
)


def looks_fulltext(text):
    """判定抓到的到底是全文，还是出版社的摘要/导航页（防误收，硬要求不虚报）。"""
    low = text.lower()
    if ('<?xml' in text[:200] or '<article' in text[:400]) and len(text) > 20000:
        return True, '全文（JATS XML %d 字符）' % len(text)
    if len(text) < 6000:
        return False, '不足6000字符'
    for mk in SUBSCRIPTION_MARKS:
        if mk in low:
            return False, '命中订阅墙标记「%s」' % mk
    heads = len(re.findall(r'(?im)^\s*(introduction|materials and methods|methods|results|'
                           r'discussion|conclusion[s]?|experimental (?:section|procedures)|'
                           r'methods and materials|background)\s*$', text))
    figs = len(re.findall(r'(?i)\b(fig(?:ure)?\.?\s*\d+)', text))
    refs = len(re.findall(r'(?im)^\s*\[?\d{1,3}[\]\.\)]\s+[A-Z]', text))
    if len(text) > 20000 and (figs >= 3 or refs >= 5):
        return True, '全文（%d字符 图%d 参考%d）' % (len(text), figs, refs)
    if heads >= 2 and (figs >= 2 or refs >= 3):
        return True, '全文（%d字符 段标题%d 图%d 参考%d）' % (len(text), heads, figs, refs)
    if heads >= 3 and len(text) > 12000:
        return True, '全文（%d字符 段标题%d）' % (len(text), heads)
    return False, '疑似摘要/导航页（%d字符 段标题%d 图%d 参考%d）' % (len(text), heads, figs, refs)


# ---- 补充通道：落地页找 PDF / Europe PMC 按 DOI / bioRxiv 预印本按标题 ----
def ch_page2pdf(landing_url):
    """落地页 → 找 PDF 链接（仓储页/出版社页通用）。"""
    try:
        html = http(landing_url, timeout=30, accept='text/html,*/*')
    except FetchError:
        return []
    urls = re.findall(r'href=["\']([^"\']+\.pdf[^"\']*)["\']', html, re.I)
    urls += re.findall(r'citation_pdf_url["\']?\s*content=["\']([^"\']+)["\']', html, re.I)
    urls += re.findall(r'["\'](https?://[^"\']*(?:/pdf/|/pdf\?|download[^"\']*\.pdf)[^"\']*)["\']', html, re.I)
    base = re.match(r'(https?://[^/]+)', landing_url)
    out = []
    for u in urls:
        if u.startswith('//'):
            u = 'https:' + u
        elif u.startswith('/') and base:
            u = base.group(1) + u
        if u.startswith('http'):
            out.append(u)
    return list(dict.fromkeys(out))[:4]


def ch_epmc_by_doi(doi):
    """Europe PMC 按 DOI 直查（修 title 匹配不到的情况）+ 预印本 PPR。"""
    out = []
    d = jget('https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=DOI:%%22%s%%22&format=json&resultType=core'
             % doi)
    for r in ((d.get('resultList') or {}).get('result') or []):
        if r.get('pmcid') and r.get('inEPMC') == 'Y':
            out.append(('https://pmc.ncbi.nlm.nih.gov/articles/%s/' % r['pmcid'], 'html', 'PMC'))
        if r.get('pmcid'):
            out.append(('https://europepmc.org/articles/%s' % r['pmcid'], 'html', 'EPMC'))
        if r.get('source') == 'PPR' and r.get('id'):
            out.append(('https://europepmc.org/api/fulltextRepo?pprId=%s&type=FILE&fileName=EMS.pdf' % r['id'], 'pdf', 'EPMC-PPR'))
    return out


def ch_biorxiv(title):
    """bioRxiv/medRxiv 预印本：Crossref 按 10.1101 前缀 + 标题反查。"""
    out = []
    if not title:
        return out
    d = jget('https://api.crossref.org/works?rows=3&filter=prefix:10.1101&query.bibliographic=%s&mailto=%s'
             % (urllib.parse.quote(title[:140]), MAIL))
    for it in (d.get('message') or {}).get('items', []) or []:
        t = ((it.get('title') or [''])[0] or '').lower()
        if len(set(re.findall(r'[a-z]{4,}', t)) & set(re.findall(r'[a-z]{4,}', title.lower()))) >= 5:
            out.append(('https://www.biorxiv.org/content/%s.full.pdf' % it.get('DOI', ''), 'pdf', 'bioRxiv'))
            out.append(('https://api.biorxiv.org/details/biorxiv/%s' % it.get('DOI', ''), 'html', 'bioRxiv-api'))
    return out


def probe_candidates(doi, meta):
    """把该 DOI 所有可能的合法全文入口全列出来（含预印本反查）。"""
    cands = []          # (url, kind, source)
    title = (meta.get('title') or '').strip()

    oa = jget('https://api.openalex.org/works/doi:%s?mailto=%s' % (doi, MAIL))
    for loc in (oa.get('locations') or []):
        if loc.get('pdf_url'):
            cands.append((loc['pdf_url'], 'pdf', 'OpenAlex-loc'))
        if loc.get('landing_page_url'):
            cands.append((loc['landing_page_url'], 'html', 'OpenAlex-land'))
    if (oa.get('best_oa_location') or {}).get('pdf_url'):
        cands.append((oa['best_oa_location']['pdf_url'], 'pdf', 'OpenAlex-best'))

    up = jget('https://api.unpaywall.org/v2/%s?email=%s' % (doi, MAIL))
    for loc in (up.get('oa_locations') or []):
        for k in ('url_for_pdf', 'url'):
            if loc.get(k):
                cands.append((loc[k], 'pdf' if k == 'url_for_pdf' else 'html', 'Unpaywall'))

    s2 = jget('https://api.semanticscholar.org/graph/v1/paper/DOI:%s?fields=openAccessPdf,title,externalIds' % doi)
    if (s2.get('openAccessPdf') or {}).get('url'):
        cands.append((s2['openAccessPdf']['url'], 'pdf', 'S2'))

    # PMC 反查：先 DOI，再标题（修 DOI 对不上的情况）
    pmc_ids = []
    ic = jget('https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=%s&format=json&tool=baorui&email=%s' % (doi, MAIL))
    for rec in (ic.get('records') or []):
        if rec.get('pmcid'):
            pmc_ids.append(rec['pmcid'])
    if title:
        q = 'TITLE:"%s"' % re.sub(r'["\[\]]', ' ', title)[:150]
        ep = jget('https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=%s&format=json&resultType=core&pageSize=3'
                  % urllib.parse.quote(q))
        for r in ((ep.get('resultList') or {}).get('result') or []):
            if r.get('pmcid'):
                pmc_ids.append(r['pmcid'])
            if r.get('source') == 'PPR' and r.get('id'):
                cands.append(('https://europepmc.org/api/fulltextRepo?pprId=%s&type=FILE&fileName=EMS.pdf' % r['id'], 'pdf', 'EPMC-PPR'))
    for pid in dict.fromkeys(pmc_ids):
        cands.append(('https://www.ebi.ac.uk/europepmc/webservices/rest/%s/fullTextXML' % pid, 'xml', 'EPMC-XML'))
        cands.append(('https://pmc.ncbi.nlm.nih.gov/articles/%s/' % pid, 'html', 'PMC-HTML'))
        cands.append(('https://europepmc.org/articles/%s' % pid, 'html', 'EPMC-HTML'))

    # 预印本反查：arXiv 标题 + bioRxiv（Europe PMC PPR 已覆盖一部分）
    if title:
        try:
            ax = http('http://export.arxiv.org/api/query?search_query=ti:%s&max_results=3'
                      % urllib.parse.quote('"%s"' % re.sub(r'[^A-Za-z0-9 ]', ' ', title)[:120]))
            for aid in re.findall(r'<id>http://arxiv.org/abs/([^<]+)</id>', ax)[:3]:
                cands.append(('https://arxiv.org/pdf/%s' % aid, 'pdf', 'arXiv'))
        except FetchError:
            pass

    for u, k, src in ch_epmc_by_doi(doi) + ch_biorxiv(title):
        cands.append((u, k, src))
    for u, k, src in list(cands):
        if k == 'html' and ('doi.org/' in u or 'springer' in u or 'nature.com' in u
                            or 'caltech' in u or 'repository' in u or 'handle' in u or 'rug.nl' in u):
            for pu in ch_page2pdf(u):
                cands.append((pu, 'pdf', src + '→PDF'))

    seen, out = set(), []
    for u, k, src in cands:
        u = u.split('#')[0]
        if u and u not in seen and not u.endswith('.xml'):
            seen.add(u)
            out.append((u, k, src))
    return out


def fetch_xml(url, doi):
    """Europe PMC fullTextXML → 去标签取正文（PMC 页 HTML 抽取不全时用这条）。"""
    try:
        xml = http(url, timeout=35, accept='application/xml,*/*')
    except FetchError as e:
        return '', 'fetch失败: %s' % str(e)[:70]
    if len(xml) < 8000:
        return '', 'XML 过短 %d' % len(xml)
    body = re.sub(r'<(ref-list|back|front)\b.*?</\1>', ' ', xml, flags=re.S)
    body = re.sub(r'<(fig|table-wrap)\b.*?</\1>', ' [图表] ', body, flags=re.S)
    body = re.sub(r'<[^>]+>', ' ', body)
    body = re.sub(r'\s{2,}', ' ', body).strip()
    return (body, '') if len(body) > 6000 else ('', 'XML 正文过短 %d' % len(body))


def fetch_text(url, kind, doi):
    """抓一个候选 URL：PDF 走 PyMuPDF；HTML 抽正文（去脚本/导航，取最长段落簇）。"""
    try:
        raw = http(url, timeout=35, binary=(kind == 'pdf'),
                   accept='application/pdf,*/*' if kind == 'pdf' else 'text/html,application/xhtml+xml')
    except FetchError as e:
        return '', 'fetch失败: %s' % str(e)[:80]
    if isinstance(raw, str):
        raw = raw.encode('utf-8', 'ignore')
    if raw[:4] == b'%PDF':
        try:
            import fitz
            doc = fitz.open(stream=raw, filetype='pdf')
            txt = '\n'.join(pg.get_text() for pg in doc)
            doc.close()
            txt = re.sub(r'\n{3,}', '\n\n', txt).strip()
            return (txt, '') if len(txt) > 4000 else ('', 'PDF 正文过短 %d' % len(txt))
        except Exception as e:
            return '', 'PDF 解析失败 %s' % str(e)[:60]
    html = raw.decode('utf-8', 'ignore')
    if len(raw) < 15000:
        return '', 'HTML 过短(可能 403 页) %d B' % len(raw)
    body = clean_html(html)
    # 去掉导航/页眉尾噪：取长度 >200 的段落里最长的连续块
    paras = [x.strip() for x in re.split(r'\n{2,}', body)]
    good = [x for x in paras if len(x) > 200]
    txt = '\n\n'.join(good)
    return (txt, '') if len(txt) > 4000 else ('', 'HTML 正文过短 %d' % len(txt))


def probe(dois, delay=0.8):
    os.makedirs(OA_DIR, exist_ok=True)
    results = []
    for i, doi in enumerate(dois, 1):
        had = os.path.exists(os.path.join(OA_DIR, fname_for(doi)))
        if had:
            print('%2d/%d ⏭  %s 已有本地全文，跳过' % (i, len(dois), doi))
            continue
        try:
            meta = ch_crossref(doi)
        except Exception:
            meta = {}
        try:
            cands = probe_candidates(doi, meta)
        except Exception as e:
            cands = []
            print('    候选枚举异常 %s' % str(e)[:80])
        print('%2d/%d %s  候选 %d 个：%s' % (i, len(dois), doi, len(cands),
              ', '.join(sorted(set(c[2] for c in cands)))))
        hit = ('', '', '', '')
        tried = 0
        for url, kind, src in cands:
            tried += 1
            txt, err = fetch_xml(url, doi) if kind == 'xml' else fetch_text(url, kind, doi)
            if txt:
                hit = (txt, src, url, kind)
                break
            print('     · %-14s %s  %s' % (src, url[:66], err))
            if tried >= 14:
                print('     · 已达单篇候选上限 14，停止试探')
                break
        if hit[0]:
            ft, why = looks_fulltext(hit[0])
            if ft:
                open(os.path.join(OA_DIR, fname_for(doi)), 'w', encoding='utf-8').write(hit[0])
                print('     ✅ 回收全文 [%s] %d 字符 · %s · %s' % (hit[1], len(hit[0]), why, hit[2][:60]))
                results.append({'doi': doi, 'status': 'OK', 'channel': hit[1], 'url': hit[2],
                                'chars': len(hit[0]), 'verify': why, 'meta': meta})
            else:
                os.makedirs(AB_DIR, exist_ok=True)
                open(os.path.join(AB_DIR, fname_for(doi)), 'w', encoding='utf-8').write(hit[0])
                print('     ⚠ 闸门拦截（%s）→ 存摘要级，不计入全文：%s' % (why, hit[2][:60]))
                results.append({'doi': doi, 'status': 'ABSTRACT', 'channel': hit[1], 'url': hit[2],
                                'chars': len(hit[0]), 'verify': why, 'meta': meta})
        else:
            print('     ⛔ 全通道失败（%d 候选）' % tried)
            results.append({'doi': doi, 'status': 'CLOSED', 'channel': '-', 'url': '',
                            'chars': 0, 'meta': meta, 'tried': tried})
        time.sleep(delay)
    json.dump(results, open(PROBE_JSON, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    ok = [r for r in results if r['status'] == 'OK']
    print('\n== 第二级汇总 ==  回收入库 %d 篇 / 探测 %d 篇' % (len(ok), len(results)))
    for r in ok:
        print('  ✅ %s [%s] %s' % (r['doi'], r['channel'], (r['meta'].get('journal') or '')[:40]))
    ab = [r for r in results if r['status'] == 'ABSTRACT']
    if ab:
        print('仅摘要级 %d 篇（不计入全文）：' % len(ab))
        for r in ab:
            print('  ⚠ %s | %s' % (r['doi'], r.get('verify', '')))
    left = [r for r in results if r['status'] == 'CLOSED']
    if left:
        print('仍封闭 %d 篇：' % len(left))
        for r in left:
            print('  - %s | %s | %s' % (r['doi'], (r['meta'].get('journal') or '')[:36], (r['meta'].get('title') or '')[:64]))
    print('结果 JSON: %s' % PROBE_JSON)


def run(dois, delay=0.6):
    os.makedirs(OA_DIR, exist_ok=True)
    results = []
    for i, doi in enumerate(dois, 1):
        have = os.path.exists(os.path.join(OA_DIR, fname_for(doi)))
        try:
            st, ch, url, text, meta, log = try_channels(doi)
        except Exception as e:          # 单篇异常不许拖垮整批
            st, ch, url, text, meta, log = 'ERROR', '-', '-', '', {}, ['异常: %s: %s' % (type(e).__name__, str(e)[:100])]
        sys.stdout.flush()
        try:
            cr = ch_crossref(doi)
        except Exception:
            cr = {}
        meta.update({k: v for k, v in cr.items() if v})
        if st == 'OK' and text:
            p = os.path.join(OA_DIR, fname_for(doi))
            open(p, 'w', encoding='utf-8').write(text)
            print('%2d/%d ✅ %s  [%s]  %d 字符' % (i, len(dois), doi, ch, len(text)))
        else:
            print('%2d/%d ⛔ %s  %s' % (i, len(dois), doi, ' / '.join(log[-3:])))
        print('        %s' % ' | '.join(log[:4]) if log else '')
        results.append({'doi': doi, 'status': st, 'channel': ch, 'url': url,
                        'had_local': have, 'chars': len(text), 'meta': meta, 'log': log})
        time.sleep(delay)
    json.dump(results, open(OUT_JSON, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    ok = [r for r in results if r['status'] == 'OK']
    new = [r for r in ok if not r['had_local']]
    print('\n== 汇总 ==  成功 %d/%d（其中新增 %d 篇）' % (len(ok), len(results), len(new)))
    closed = [r for r in results if r['status'] != 'OK']
    if closed:
        print('仍封闭 %d 篇：' % len(closed))
        for r in closed:
            print('  - %s | %s | %s' % (r['doi'], (r['meta'].get('journal') or '')[:40], (r['meta'].get('title') or '')[:70]))
    print('结果 JSON: %s' % OUT_JSON)


if __name__ == '__main__':
    cmd = sys.argv[1] if len(sys.argv) > 1 else 'report'
    if cmd == 'get':
        run(sys.argv[2:])
    elif cmd == 'list':
        dois = [l.strip() for l in open(sys.argv[2], encoding='utf-8') if l.strip() and not l.startswith('#')]
        run(dois)
    elif cmd == 'probe':
        dois = [l.strip() for l in open(sys.argv[2], encoding='utf-8') if l.strip() and not l.startswith('#')]
        probe(dois)
    else:
        if os.path.exists(OUT_JSON):
            rs = json.load(open(OUT_JSON, encoding='utf-8'))
            ok = [r for r in rs if r['status'] == 'OK']
            print('上次运行：%d 篇，成功 %d，新增 %d' % (len(rs), len(ok), len([r for r in ok if not r['had_local']])))
            for r in ok:
                print('  ✅ %s [%s]' % (r['doi'], r['channel']))
        else:
            print('无结果文件')