#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""抓取诺唯赞官网（bio.vazyme.com）分子诊断/原料产品信息。

背景：`/product/<id>`（无后缀）被 WAF 拦（403, CE-WAF-WLF），实测 `/product/<id>.html` 可 200。
策略：从首页取导航 → 逐类目页收集产品链接（用站点自身 href 形态）→ 逐个详情页取
      品名 / 货号 / 规格 / 关键描述，落盘 JSON，正文存文本便于人工核对。
仅抓公开页面，限速 1.2s，失败重试一次（换 .html 后缀 / 加尾斜杠）。
"""
import json
import os
import re
import time
import urllib.parse
import urllib.request

from lxml import html as LH

BASE = 'https://bio.vazyme.com'
UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
      '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36')
W = '/Users/liuxinyuan/Desktop/Hermes输出-工作类'
OUT = os.path.join(W, '数据/competitor_vazyme.json')
RAWDIR = os.path.expanduser('~/.hermes/profiles/zone-b/cache/scratch/competitor_raw/vazyme')
COOKIE = {'v': ''}


def fetch(url, referer=BASE + '/', tries=3):
    for i in range(tries):
        req = urllib.request.Request(url, headers={
            'User-Agent': UA, 'Referer': referer, 'Accept-Language': 'zh-CN,zh;q=0.9',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            **({'Cookie': COOKIE['v']} if COOKIE['v'] else {})})
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                raw = r.read()
                sc = r.headers.get('Set-Cookie')
                if sc and not COOKIE['v']:
                    COOKIE['v'] = sc.split(';')[0]
                return r.geturl(), raw.decode('utf-8', 'ignore')
        except Exception as e:
            if i == tries - 1:
                return None, 'ERR:%s' % str(e)[:80]
            time.sleep(1.5)
    return None, ''


def text_of(html_bytes):
    t = LH.fromstring(html_bytes)
    for bad in t.xpath('//script|//style'):
        bad.getparent().remove(bad)
    return re.sub(r'\n{2,}', '\n', re.sub(r'[ \t]+', ' ', t.text_content())).strip()


def main():
    os.makedirs(RAWDIR, exist_ok=True)
    url, h = fetch(BASE + '/')
    if not h or h.startswith('ERR'):
        print('首页失败', h); return
    home = text_of(h)
    links = sorted({urllib.parse.urljoin(BASE + '/', a.get('href'))
                    for a in LH.fromstring(h).xpath('//a[@href]')})
    cats = [u for u in links if re.search(r'/(products|product)/[\w.]+', u)]
    cats = sorted(set(cats), key=lambda u: (('原料' not in u), u))[:60]
    print('首页发现链接 %d ｜ 候选类目/产品 %d' % (len(links), len(cats)))

    # 目标类目：分子诊断 / 原料 / 酶相关
    KEY = ('分子诊断', '原料', '酶', 'PCR', '逆转录', 'LAMP', '冻干', 'qPCR')
    prod_links, seen = [], set()
    for u in cats:
        _, ch = fetch(u)
        if not ch or ch.startswith('ERR'):
            continue
        ct = text_of(ch)
        if not any(k in ct for k in KEY):
            continue
        ps = sorted({urllib.parse.urljoin(u, a.get('href'))
                     for a in LH.fromstring(ch).xpath('//a[@href]')
                     if re.search(r'/product/\d+', a.get('href') or '')})
        new = [p for p in ps if p not in seen]
        seen.update(new)
        prod_links.extend(new)
        print('  类目 %s ｜ 产品链接 +%d' % (u.replace(BASE, '')[:40], len(new)))
        time.sleep(1.2)
    print('合计产品链接 %d' % len(prod_links))

    items = []
    for i, u in enumerate(prod_links, 1):
        final, ph = fetch(u, referer=u)
        if not ph or ph.startswith('ERR'):
            alt = u.rstrip('/') + '.html' if not u.endswith('.html') else u.rstrip('.html')
            final, ph = fetch(alt, referer=alt)
        if not ph or ph.startswith('ERR'):
            items.append({'url': u, '错误': str(ph)[:60]})
            continue
        body = text_of(ph)
        tm = re.search(r'<title>([^<]{2,80})</title>', ph)
        name = (tm.group(1).strip() if tm else '')
        codes = sorted({c for c in re.findall(r'\b[A-Z]{2}\d{3,4}[A-Za-z0-9\-]*\b', body)})
        items.append({'url': final, '页面标题': name, '货号': codes[:12], '正文': body[:3000]})
        fn = re.sub(r'[^\w.\-]+', '_', (final or u).replace(BASE, ''))[:80] + '.txt'
        open(os.path.join(RAWDIR, fn), 'w', encoding='utf-8').write(body[:8000])
        if i % 10 == 0:
            print('  详情 %d/%d' % (i, len(prod_links)))
        time.sleep(1.2)

    ok = [x for x in items if '错误' not in x]
    json.dump({'抓取时间': time.strftime('%Y-%m-%d %H:%M'), '来源': BASE,
               '口径': '公开产品页；/product/<id> 被 WAF 拦，改走 <id>.html 与站点自身链接',
               '条数': len(items), '成功': len(ok), 'items': items},
              open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    print('\n成功 %d / %d → %s' % (len(ok), len(items), OUT))
    for x in ok[:10]:
        print('  ✓', x['页面标题'][:44], '｜货号', x['货号'][:4])


if __name__ == '__main__':
    main()