# -*- coding: utf-8 -*-
"""抓取并归档原始资料（HTML 正文），供分子诊断市场取证使用。
用法: python3 fetch_archive.py <url> <slug>
输出: 资料/文献与专利/2025-2027/market/<YYYY-MM-DD>_<slug>.html 与 .txt
"""
import sys, os, re, json, datetime, hashlib
import urllib.request

BASE = "/Users/liuxinyuan/Desktop/Hermes输出-工作类/资料/文献与专利/2025-2027/market"
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")


def fetch(url, timeout=45):
    req = urllib.request.Request(url, headers={
        "User-Agent": UA,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    })
    with urllib.request.urlopen(req, timeout=timeout) as r:
        raw = r.read()
        ctype = r.headers.get("Content-Type", "")
    for enc in ("utf-8", "gb18030", "latin-1"):
        try:
            return raw.decode(enc), ctype
        except UnicodeDecodeError:
            continue
    return raw.decode("utf-8", "ignore"), ctype


def to_text(html):
    h = re.sub(r"(?is)<(script|style|noscript|svg)[^>]*>.*?</\1>", " ", html)
    h = re.sub(r"(?is)<br\s*/?>", "\n", h)
    h = re.sub(r"(?is)</(p|div|li|tr|h[1-6]|table|section)>", "\n", h)
    h = re.sub(r"(?is)<[^>]+>", " ", h)
    import html as _h
    h = _h.unescape(h)
    h = re.sub(r"[ \t\u00a0]+", " ", h)
    h = re.sub(r"\n\s*\n+", "\n", h)
    return h.strip()


def main():
    url, slug = sys.argv[1], sys.argv[2]
    os.makedirs(BASE, exist_ok=True)
    day = datetime.date.today().isoformat()
    try:
        body, ctype = fetch(url)
    except Exception as e:
        print(json.dumps({"ok": False, "url": url, "err": repr(e)}, ensure_ascii=False))
        return
    stem = f"{day}_{slug}"
    with open(os.path.join(BASE, stem + ".html"), "w", encoding="utf-8") as f:
        f.write(f"<!-- SOURCE: {url}\n     FETCHED: {datetime.datetime.now().isoformat()}\n     CTYPE: {ctype} -->\n")
        f.write(body)
    txt = to_text(body)
    with open(os.path.join(BASE, stem + ".txt"), "w", encoding="utf-8") as f:
        f.write(f"SOURCE: {url}\nFETCHED: {datetime.datetime.now().isoformat()}\n\n{txt}")
    print(json.dumps({"ok": True, "url": url, "file": stem,
                      "bytes": len(body), "text_chars": len(txt),
                      "sha1": hashlib.sha1(body.encode('utf-8', 'ignore')).hexdigest()[:12]},
                     ensure_ascii=False))


if __name__ == "__main__":
    main()