#!/usr/bin/env python3 # -*- coding: utf-8 -*- """把抓到的诺唯赞官网产品页解析成竞品条目（品名 / 货号 / 规格 / 卖点 / 来源）。 输入：数据/competitor_vazyme_products.json（56 页，WAF 绕过抓取） 输出：数据/competitor_vazyme_entries.json（结构化条目） """ import json import os import re W = '/Users/liuxinyuan/Desktop/Hermes输出-工作类' SRC = os.path.join(W, '数据/competitor_vazyme_products.json') OUT = os.path.join(W, '数据/competitor_vazyme_entries.json') # 诺唯赞货号形态（实测）：R301 / RM1022-01 / Q513-02 / QL101-01 / RL301-01 / CL101-01 / C116-01 / DL2000 / GR501-01 / JC3001-00 / CG101 / JM2001 / RM3876 CODE_OK = re.compile(r"^[A-Z]{2,4}\d{2,5}(?:-\d{2,3}){0,2}$") CODE_BAD = {"DNA", "PCR", "RNA", "CT", "ELISA"} TYPES = [ ('克隆/建库/电泳', r'Cloning|ClonExpress|Ligation|TOPO|Marker|Ladder|GelRed|TBE|连接酶|内切酶'), ('LAMP/等温扩增', r'LAMP|Bst|等温|Isothermal'), ('qPCR/RT-qPCR 预混液', r'qPCR|qRT-PCR|SuperMix|RT-PCR Kit|预混|Master Mix'), ('反转录酶', r'Reverse Transcriptase|逆转录|反转录|RT'), ('热启动 Taq / DNA 聚合酶', r'Taq|Polymerase|聚合酶|高保真'), ('UDG/UNG 酶', r'UDG|UNG|Uracil'), ('RNase 抑制剂', r'RNase|RNA酶'), ('封闭/阻断原料', r'Blocker|封闭|阻断'), ('蛋白/其他原料', r'蛋白|Protein|Nuclease|核酸酶|内切酶|连接酶|染料|Marker|磁珠|微球|填料'), ] TAGS = [('可冻干', r'Lyo|可冻干|冻干'), ('无甘油', r'Glycerol-?free|无甘油')] NAV_NOISE = ('首页', '产品中心', '分子生物学', '核酸提取与纯化', '细胞生物学', '蛋白生物学', '高通量试剂', '分子诊断试剂', '仪器设备', '病原检测', '生物耗材', '原料和解决方案', '生物标志物检测', 'PCR', '核酸电泳', '克隆', '逆转录', 'qPCR', 'RT-qPCR热启动系列', 'DNA提取', 'RNA提取', 'DNA/RNA共提取', '核酸提取相关试剂', '外泌体分离试剂', '自动化核酸提取仪', '细胞分选', '细胞培养', '污染检测与清除', '细胞转染', '细胞凋亡检测', '细胞增殖与毒性检测', '基因表达与调控', '细胞衰老检测', '3D细胞培养', '细胞周期检测', '蛋白纯化', '蛋白互作', '病理免疫', '蛋白修饰', '免疫印迹', '抗体', 'DNA建库系列', 'RNA建库系列', '多重扩增系列', '表观遗传系列', '单细胞系列', '三代建库系列', '杂交捕获系列', '生物信息', '服务', '关于我们', '联系我们', '技术资源', '资料下载', '促销活动', '新闻中心', '加入我们', '在线商城', 'EN') def clean_title(t): t = t.replace('_产品中心_南京诺唯赞生物科技股份有限公司', '') t = re.sub(r'&', '&', t) parts = [p for p in t.split('_') if p.strip()] return parts[0].strip() if parts else t.strip() def _cell_codes(cell): """从单元格里抓货号（含列头形态：QL101-01 1,000 U (5 U/μl)）。""" out = [] for tok in re.findall(r"[A-Z]{2,4}\d{2,5}(?:-\d{2,3}){0,2}", str(cell or "")): if CODE_OK.match(tok) and tok not in CODE_BAD: out.append(tok) return out def codes_from(item, name=""): hdr_codes = [] for tb in item.get("表格") or []: if not tb: continue for c in tb[0][1:]: hdr_codes.extend(_cell_codes(c)) if hdr_codes: out, seen = [], set() for c in hdr_codes: if c not in seen: seen.add(c) out.append(c) return out[:8] """只取「本产品」的货号：表格含货号列且该行名称与品名相关；否则留空待补。""" out = [] nn = re.sub(r"[^A-Za-z\u4e00-\u9fff]", "", name)[:6] for tb in item.get("表格") or []: if not tb: continue hdr = [str(c) for c in tb[0]] ci = next((i for i, h in enumerate(hdr) if "货号" in h), None) ni = next((i for i, h in enumerate(hdr) if "名称" in h or "产品" in h), None) if ci is None: continue for row in tb[1:]: if ci >= len(row) or not row[ci]: continue if ni is not None and ni < len(row) and row[ni]: rn = re.sub(r"[^A-Za-z\u4e00-\u9fff]", "", str(row[ni])) if nn and nn[:4] not in rn and rn[:4] not in re.sub(r"[^A-Za-z\u4e00-\u9fff]", "", name): continue for c in re.split(r"[、,，/\s]+", str(row[ci])): if CODE_OK.match(c.strip()): out.append(c.strip()) seen, uniq = set(), [] for c in out: if c not in seen: seen.add(c) uniq.append(c) return uniq[:8] def specs_from(item): sp = [] for tb in item.get("表格") or []: if not tb: continue # 列头里常写「货号 规格」：QL101-01 1,000 U (5 U/μl) for c in tb[0][1:]: m = re.sub(r"^[A-Z]{2,4}\d{2,5}(?:-\d{2,3}){0,2}\s*", "", str(c or "")).strip() if m and len(m) < 40: sp.append(m) hdr = [str(c) for c in tb[0]] ci = next((i for i, h in enumerate(hdr) if '规格' in h), None) if ci is None: continue for row in tb[1:]: if ci < len(row) and row[ci]: sp.append(str(row[ci]).strip()) return sp[:4] def points_from(item): """从正文里摘出非导航区的描述句（含产品/特性/适用/推荐 等关键词）。""" txt = item.get('正文') or '' lines = [re.sub(r'\s+', ' ', l).strip() for l in txt.split('\n')] pts = [] for l in lines: if not l or l in NAV_NOISE or len(l) < 8 or len(l) > 160: continue if l.startswith(('首页', '产品中心', 'window', 'var ', 'function')) or '{' in l: continue if re.search(r'(产品|特性|特点|适用|推荐|用于|灵敏度|特异性|稳定性|浓度|活性|冻干|无甘油|Glycerol|性能|可|支持|兼容)', l): pts.append(l) # 去重 seen, uniq = set(), [] for p in pts: k = p[:24] if k not in seen: seen.add(k) uniq.append(p) return uniq[:5] def main(): d = json.load(open(SRC, encoding='utf-8')) ents, seen = [], set() for it in d['items']: title = clean_title(it.get('页面标题') or '') if not title or '产品中心' in title or '生物耗材' in title: continue name = title.split('（')[0].split('(')[0].strip() if not name or name in seen: continue seen.add(name) blob = name + ' ' + ' '.join(it.get('货号') or []) cat = next((c for c, pat in TYPES if re.search(pat, blob, re.I)), '其他') tags = [t for t, pat in TAGS if re.search(pat, title, re.I)] ents.append({'品牌': '诺唯赞', '货号': ' / '.join(codes_from(it, title)) or '待补（页面未列）', '品名': title, '品类': cat, '标签': tags, '规格': '；'.join(specs_from(it)), '卖点': points_from(it), '来源': it.get('url', ''), '数据类型': '官网产品页（路径级 WAF 绕过：.html/尾斜杠）'}) json.dump({'生成时间': __import__('pandas').Timestamp.now().strftime('%Y-%m-%d %H:%M'), '来源': 'https://bio.vazyme.com', '条数': len(ents), '口径': '诺唯赞官网产品页公开信息；/product/ 被 WAF 拦，改走 .html 与尾斜杠路径抓取', 'items': ents}, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False, indent=1) print('诺唯赞条目 %d 条' % len(ents)) for e in ents: print(' %-42s ｜ %-18s ｜ %-22s ｜ %s' % (e['品名'][:42], e['货号'][:18], e['品类'][:22], '+'.join(e['标签']))) print('\n→', OUT) if __name__ == '__main__': main()