#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""诺唯赞第二轮：精准抓「分子诊断试剂 / 原料和解决方案 / 酶原料」类目下的产品。

导航按链接文字筛选（分子诊断|原料|酶|PCR|逆转录|LAMP|qPCR|冻干），再下钻一层到产品页，
抽取 品名 / 货号 / 规格与参数表格。绕过 WAF：优先用站点自身 href，失败改 .html / 尾斜杠。
"""
import json
import os
import re
import time
import urllib.parse
import urllib.request

from lxml import html as LH

BASE = 'https://bio.vazyme.com'
UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
      '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36')
W = '/Users/liuxinyuan/Desktop/Hermes输出-工作类'
OUT = os.path.join(W, '数据/competitor_vazyme_products.json')
RAW = os.path.expanduser('~/.hermes/profiles/zone-b/cache/scratch/competitor_raw/vazyme2')
NAV_KEY = ('分子诊断', '原料', '酶', 'PCR', '逆转录', 'LAMP', 'qPCR', '冻干', '测序', '核酸提取')
CODE_RE = re.compile(r'\b(?:[A-Z]{1,3}\d{2,5}(?:-\d{1,4}){0,2})\b')


def fetch(url, referer=BASE + '/'):
    for i in range(3):
        req = urllib.request.Request(url, headers={
            'User-Agent': UA, 'Referer': referer, 'Accept-Language': 'zh-CN,zh;q=0.9',
            'Accept': 'text/html,application/xhtml+xml,*/*;q=0.8'})
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return r.geturl(), r.read().decode('utf-8', 'ignore')
        except Exception:
            time.sleep(1.5)
    return None, ''


def alt_urls(u):
    if u.endswith('.html'):
        yield u
        yield u[:-5] + '/'
    elif u.endswith('/'):
        yield u
        yield u.rstrip('/') + '.html'
    else:
        yield u
        yield u + '.html'
        yield u + '/'


def get_ok(u):
    for cand in alt_urls(u):
        final, h = fetch(cand, referer=BASE + '/')
        if h and not h.lstrip().startswith('<!DOCTYPE html>\n<html lang="en">'):
            return final, h
    return None, ''


def clean(h):
    t = LH.fromstring(h)
    for bad in t.xpath('//script|//style'):
        bad.getparent().remove(bad)
    txt = re.sub(r'\n{2,}', '\n', re.sub(r'[ \t]+', ' ', t.text_content())).strip()
    tables = []
    for tb in t.xpath('//table'):
        rows = [[re.sub(r'\s+', ' ', (c.text_content() or '')).strip() for c in tr.xpath('./td|./th')]
                for tr in tb.xpath('.//tr')]
        rows = [r for r in rows if any(r)]
        if len(rows) >= 2:
            tables.append(rows)
    return txt, tables


def main():
    os.makedirs(RAW, exist_ok=True)
    _, home = get_ok(BASE + '/')
    tree = LH.fromstring(home)
    navs = []
    for a in tree.xpath('//a[@href]'):
        label = re.sub(r'\s+', '', a.text_content() or '')
        href = urllib.parse.urljoin(BASE + '/', a.get('href'))
        if not label or not re.search(r'/(products?_?[\w.]*|products/\d+\.html|product/\d+/?)$', href):
            continue
        if any(k in label for k in NAV_KEY):
            navs.append((label, href))
    # 去重 + 补已知类目
    seen, cats = set(), []
    for label, href in navs + [('分子诊断-冻干试剂', BASE + '/products_29.html'),
                               ('分子诊断-一步法RT-PCR', BASE + '/products_31.html'),
                               ('原料和解决方案', BASE + '/product/262/')]:
        if href not in seen:
            seen.add(href)
            cats.append((label, href))
    print('目标类目 %d 个:' % len(cats))
    for l, h in cats:
        print('   %-24s %s' % (l[:24], h.replace(BASE, '')))

    prods, seenp = [], set()
    for label, url in cats:
        final, h = get_ok(url)
        if not h:
            print('  ✗ 类目失败', label, url.replace(BASE, ''))
            continue
        txt, _ = clean(h)
        links = []
        for a in LH.fromstring(h).xpath('//a[@href]'):
            href = urllib.parse.urljoin(url, a.get('href'))
            if re.search(r'/product/\d+', href):
                nm = re.sub(r'\s+', ' ', a.text_content() or '').strip()
                links.append((nm, href))
        new = [(n, x) for n, x in links if x not in seenp]
        for n, x in new:
            seenp.add(x)
        prods.extend([(label, n, x) for n, x in new])
        print('  ✓ %-22s 产品链接 +%d（累计 %d）' % (label[:22], len(new), len(prods)))
        time.sleep(1.2)

    items, ok = [], 0
    for i, (cat, nm, url) in enumerate(prods, 1):
        final, h = get_ok(url)
        if not h:
            items.append({'类目': cat, 'url': url, '错误': 'fetch failed'})
            continue
        txt, tables = clean(h)
        tm = re.search(r'<title>([^<]{2,90})</title>', h)
        title = (tm.group(1).strip() if tm else nm)
        codes = sorted({c for c in CODE_RE.findall(txt) if any(ch.isdigit() for ch in c)})[:14]
        items.append({'类目': cat, '页面标题': title, 'url': final, '货号': codes,
                      '表格': tables[:4], '正文': txt[:2500]})
        ok += 1
        fn = re.sub(r'[^\w.\-]+', '_', final.replace(BASE, ''))[:80] + '.txt'
        open(os.path.join(RAW, fn), 'w', encoding='utf-8').write(txt[:9000])
        if i % 10 == 0:
            print('   详情 %d/%d' % (i, len(prods)))
        time.sleep(1.2)

    json.dump({'抓取时间': time.strftime('%Y-%m-%d %H:%M'), '来源': BASE, '类目数': len(cats),
               '条数': len(items), '成功': ok, 'items': items},
              open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1)
    print('\n成功 %d / %d → %s' % (ok, len(items), OUT))


if __name__ == '__main__':
    main()