#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""行业信息推送 v0.1 采集器（试跑版：只落档，不推送）。

三路：
  R1 巨潮公告（POST）—— 9 家上市客户体系 + 诺唯赞
  R2 竞品官网新闻 —— 诺唯赞/翌圣/全式金/赛默飞/罗氏/菲鹏
  R3 招投标 —— 由浏览器另跑，结果由本脚本并入（runs/<date>_ggzy.json）

输出：数据/market_intel/runs/<YYYYMMDD>_items.jsonl + <YYYYMMDD>_v01试跑.md
"""
import json
import os
import re
import sys
import time
from datetime import datetime, timedelta

import requests
from bs4 import BeautifulSoup

W = '/Users/liuxinyuan/Desktop/Hermes输出-工作类'
CFG = json.load(open(os.path.join(W, '数据/market_intel/v01/sources_v01.json'), encoding='utf-8'))
LISTED = json.load(open(os.path.join(W, '数据/market_intel/客户上市状态_2026-10-13.json'), encoding='utf-8'))
SUP = json.load(open(os.path.join(W, '数据/market_intel/客户上市状态_补测.json'), encoding='utf-8'))
RUNS = os.path.join(W, '数据/market_intel/runs')
# ── 时间窗：首次调研 = 近三年；每日推送 = 仅前一天 ──
MODE = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] in ('first', 'daily') else 'daily'
WIN_DAYS = CFG['时间逻辑']['首次调研']['窗口天数'] if MODE == 'first' else CFG['时间逻辑']['每日推送']['窗口天数']
WIN_FROM = (datetime.now() - timedelta(days=WIN_DAYS)).strftime('%Y-%m-%d')
TARGET_DAY = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
os.makedirs(RUNS, exist_ok=True)
TODAY = datetime.now().strftime('%Y%m%d')
UA = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
      '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36')
S = requests.Session()
S.headers.update({'User-Agent': UA, 'Accept-Language': 'zh-CN,zh;q=0.9'})
TAGS = CFG['命中标签']
items, status = [], []


def tag_of(text):
    hits = []
    for cat, kws in TAGS.items():
        for k in kws:
            if k.lower() in text.lower():
                hits.append('%s:%s' % (cat, k))
    return list(dict.fromkeys(hits))


def add(src, brand, title, url, date='', extra=''):
    title = re.sub(r'\s+', ' ', title).strip()
    if not title:
        return
    items.append({'来源': src, '主体': brand, '标题': title[:180], '日期': date,
                  '链接': url, '标签': tag_of(title + ' ' + extra), '抓取时间': datetime.now().isoformat(timespec='seconds')})


# ─────────── R1 巨潮公告 ───────────
def r1_cninfo():
    ents = [('688105', '9900046657', '诺唯赞')]
    EXCLUDE = {'佰奥莱博集团',  # 300836 佰奥智能为误匹配，禁用
               '海尔施集团'}      # 实测：601206 巨潮仅 6 条公告且全为 2016-08 IPO 文件
                                 #（最新一条为「首次公开发行A股暂缓发行」）→ IPO 暂缓后无披露，非有效监测标的
    for nm, v in LISTED.items():
        if nm in EXCLUDE:
            continue
        if v.get('上市') and v['匹配']:
            m = v['匹配'][0]
            ents.append((m['代码'], m.get('orgId'), nm))
    for nm, hits in SUP.items():
        if hits and nm not in [e[2] for e in ents] and nm not in EXCLUDE:
            ents.append((hits[0][0], None, nm))

    def orgid_of(code):
        try:
            r = S.post('http://www.cninfo.com.cn/new/information/topSearch/query',
                       data={'keyWord': code, 'maxNum': 5},
                       headers={'X-Requested-With': 'XMLHttpRequest'}, timeout=20)
            for it in (r.json() or []):
                if str(it.get('code')) == str(code):
                    return it.get('orgId')
        except Exception:
            pass
        return None
    seen = set()
    ok = 0
    for code, org, nm in ents:
        key = (code, nm)
        if key in seen:
            continue
        seen.add(key)
        if not org:
            org = orgid_of(code)
            time.sleep(0.8)
        stock = '%s,%s' % (code, org) if org else code
        try:
            got, page, cap = 0, 1, (1500 if MODE == 'first' else 30)   # 首次模式：按时间窗翻到底（上限仅作安全阀）
            while page <= 200:
                r = S.post(CFG['巨潮公告']['endpoint'],
                           data={'stock': stock, 'tabName': 'fulltext', 'pageSize': 30, 'pageNum': page,
                                 'column': 'sse', 'category': ''},
                           headers={'X-Requested-With': 'XMLHttpRequest',
                                    'Referer': 'http://www.cninfo.com.cn/'}, timeout=25)
                j = r.json()
                ann = j.get('announcements') or []
                if not ann:
                    break
                stop = False
                for a in ann:
                    t = int(a.get('announcementTime') or 0) / 1000
                    d = datetime.fromtimestamp(t).strftime('%Y-%m-%d') if t else ''
                    if d and d < WIN_FROM:
                        stop = True
                        continue
                    if MODE == 'daily' and d != TARGET_DAY:
                        continue
                    url = 'http://static.cninfo.com.cn/' + str(a.get('adjunctUrl', ''))
                    add('巨潮公告', nm, a.get('announcementTitle', ''), url, d, '公告')
                    got += 1
                    if got >= cap:
                        stop = True
                        break
                if stop or len(ann) < 30:
                    break
                page += 1
                time.sleep(1.0)
            ok += 1
            print('  R1 %-28s %-8s 窗口内 %d 条（模式 %s，总数 %s）' % (
                nm[:28], code, got, MODE, j.get('totalAnnouncement')))
        except Exception as e:
            print('  R1 %-28s ❌ %s' % (nm[:28], str(e)[:60]))
        time.sleep(1.1)
    status.append(('R1 巨潮公告', '%d/%d 主体成功' % (ok, len(seen)), 'live'))


# ─────────── R2 竞品官网新闻 ───────────
def r2_news():
    """竞品官网新闻。窗口：first = 近三年（可翻页的翻页）；daily = 仅前一日。"""
    for src in CFG['竞品官网新闻']:
        brand, url = src['品牌'], src['列表']
        if '备注' in src and '不采集' in src.get('备注', ''):
            status.append(('R2 竞品官网 · ' + brand, src['备注'], 'skip'))
            continue
        n = 0
        try:
            # 需要翻页的源（首次模式）
            page_urls = [url]
            if MODE == 'first' and 'transgen.com/news/list_' in url:
                page_urls = ['https://www.transgen.com/news/list_%d.html' % k for k in range(1, 13)]
            for page_url in page_urls:
                r = S.get(page_url, timeout=30)
                r.encoding = r.apparent_encoding or r.encoding
                soup = BeautifulSoup(r.text, 'lxml')

                if '条目选择器' in src:
                    for it in soup.select(src['条目选择器']):
                        a = it.select_one(src['标题选择器']) or it.find('a', href=True)
                        if not a:
                            continue
                        href = a.get('href', '')
                        if href.startswith('/'):
                            href = re.match(r'https?://[^/]+', page_url).group(0) + href
                        dt = it.select_one(src['日期选择器'])
                        dd = dt.get_text(strip=True) if dt else ''
                        if dd and (dd < WIN_FROM or (MODE == 'daily' and dd != TARGET_DAY)):
                            continue
                        add('竞品官网 · ' + brand, brand, a.get_text(' ', strip=True), href, dd)
                        n += 1

                elif page_url.endswith('.xml'):
                    pairs = re.findall(r'<loc>\s*([^<\s]+)\s*</loc>\s*(?:<lastmod>\s*([^<\s]+)\s*</lastmod>)?', r.text)
                    got = [(l, d) for l, d in pairs if re.search(src['链接模式'], l)]
                    got = [(l, d) for l, d in got if not d or d >= WIN_FROM]
                    got.sort(key=lambda x: x[1] or '', reverse=True)
                    if MODE == 'daily':
                        day = [x for x in got if x[1] == TARGET_DAY]
                        got = day if day else got[:14]
                    for loc, lm in got[:(200 if MODE == 'first' else 14)]:
                        slug = loc.rstrip('/').split('/')[-1].replace('.html', '').replace('-', ' ')
                        add('竞品官网 · ' + brand, brand, slug, loc, lm or '')
                        n += 1

                elif '链接模式' in src:
                    pat = re.compile(src['链接模式'])
                    multi = len(page_urls) > 1
                    for a in soup.find_all('a', href=True):
                        if not pat.search(a['href']):
                            continue
                        txt = a.get_text(' ', strip=True)
                        if len(txt) < 8:
                            continue
                        dm = re.search(r'20\d{2}-\d{2}-\d{2}', txt + ' ' + a['href'])
                        dd = dm.group(0) if dm else ''
                        if dd and (dd < WIN_FROM or (MODE == 'daily' and dd != TARGET_DAY)):
                            continue
                        href = a['href']
                        if href.startswith('/'):
                            href = re.match(r'https?://[^/]+', page_url).group(0) + href
                        add('竞品官网 · ' + brand, brand, txt, href, dd)
                        n += 1
                        if n >= (200 if MODE == 'first' else 14):
                            break
                    if multi:
                        time.sleep(0.8)
                if n >= (200 if MODE == 'first' else 14):
                    break
            status.append(('R2 竞品官网 · ' + brand, '取到 %d 条' % n, 'live' if n else 'empty'))
            print('  R2 %-14s %d 条' % (brand, n))
        except Exception as e:
            status.append(('R2 竞品官网 · ' + brand, '失败: %s' % str(e)[:60], 'fail'))
            print('  R2 %-14s ❌ %s' % (brand, str(e)[:60]))
        time.sleep(1.3)



def r3_ggzy():
    kws = CFG['招投标']['关键词']
    # 3a 浏览器渲染的 ggzy 列表（v0.1 暂缺）
    p = os.path.join(RUNS, TODAY + '_ggzy.json')
    n_gz = 0
    if os.path.exists(p):
        d = json.load(open(p, encoding='utf-8'))
        for it in d.get('items', []):
            txt = it.get('标题', '') + ' ' + it.get('地区', '')
            if any(k in txt for k in kws) or tag_of(txt):
                add('全国公共资源交易平台', it.get('地区', '全国'), it.get('标题', ''), it.get('链接', ''),
                    it.get('日期', ''), it.get('类型', ''))
                n_gz += 1
        status.append(('R3a 招投标 ggzy', '命中 %d / %d' % (n_gz, len(d.get('items', []))), 'live'))
    else:
        status.append(('R3a 招投标 ggzy', '列表页 JS 渲染 + 搜索 POST 405 + getTradList 404 → v0.2 接入', 'pending'))
    # 3b 检索源兜底
    q = os.path.join(RUNS, TODAY + '_search.json')
    n_se = 0
    if os.path.exists(q):
        d = json.load(open(q, encoding='utf-8'))
        for it in d.get('items', []):
            txt = it.get('标题', '') + ' ' + it.get('摘要', '')
            if any(k in txt for k in kws) or tag_of(txt):
                add('检索源 · 招投标', '招投标', it.get('标题', ''), it.get('链接', ''),
                    it.get('日期', ''), it.get('检索词', ''))
                n_se += 1
        status.append(('R3b 检索源（招投标兜底）', '命中 %d / %d' % (n_se, len(d.get('items', []))), 'live'))
    print('  R3 ggzy %d 条 ｜ 检索源 %d 条' % (n_gz, n_se))


print('══ v0.1 采集开始 %s ══' % datetime.now().strftime('%Y-%m-%d %H:%M'))
r1_cninfo()
r2_news()
r3_ggzy()

# 去重（同链接保留首条）
seen, uniq = set(), []
for it in items:
    k = (it['链接'], it['标题'][:40])
    if k in seen:
        continue
    seen.add(k)
    uniq.append(it)
items = uniq

SUF = '_items_first.jsonl' if MODE == 'first' else '_items.jsonl'
with open(os.path.join(RUNS, TODAY + SUF), 'w', encoding='utf-8') as f:
    for it in items:
        f.write(json.dumps(it, ensure_ascii=False) + '\n')

print('\n══ 汇总 ══')
print('  条目总数:', len(items))
from collections import Counter
print('  按来源:', dict(Counter(i['来源'] for i in items).most_common()))
print('  带标签条目:', sum(1 for i in items if i['标签']))
json.dump({'run': TODAY, 'mode': MODE, '窗口起': WIN_FROM, 'items': len(items), 'status': status},
          open(os.path.join(RUNS, TODAY + ('_status_first.json' if MODE == 'first' else '_status.json')), 'w', encoding='utf-8'),
          ensure_ascii=False, indent=1)
for s in status:
    print('   [%s] %-26s %s' % (s[2].upper(), s[0], s[1]))