#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""IMA 诊断原料知识库语料构建
（仅两个业务知识库的公开条目；不读取个人笔记）
B. 微信文章：无 URL，按标题检索（DeepSeek Search）-> 命中则 Fetch 正文
C. 本地副本：直接抽取文本（PDF/xlsx/md/html）
输出：数据/ima_corpus.json
"""
import datetime as dt
import difflib
import json
import os
import re
import sys
import time

import requests

W = '/Users/liuxinyuan/Desktop/Hermes输出-工作类'
sys.path.insert(0, os.path.join(W, '脚本'))
import deepseek_search as DS  # noqa: E402

CUTOFF = dt.date(2026, 7, 17)
items = []




# ---------- B. 微信文章标题检索 ----------
print('[B] 微信文章标题检索')
idx = json.load(open(os.path.join(W, '数据/ima_kb_index_诊断原料.json'), encoding='utf-8'))
wx = [x for x in idx['items'] if x['media_type'] == 6]
print('  待检索', len(wx))
for i, x in enumerate(wx, 1):
    t = (x['title'] or '').strip()
    rec = {'source': 'wechat', 'title': t, 'path': x['path'], 'date': '', 'status': 'miss', 'url': '', 'text': ''}
    try:
        rs = DS.search('"%s"' % t[:48], count=3)
        best, score = None, 0.0
        for r in rs:
            s = difflib.SequenceMatcher(None, t, (r.get('title') or '')).ratio()
            if s > score:
                best, score = r, s
        if best and score >= 0.6:
            rec['url'] = best['url']
            rec['date'] = (best.get('date') or '')[:10]
            body = best.get('content') or ''
            if len(body) < 300:
                try:
                    body = DS.fetch(best['url']).get('content', '')
                except Exception:
                    body = best.get('summary') or ''
            rec['text'] = re.sub(r'\s+', ' ', body)[:6000]
            rec['status'] = 'ok' if len(rec['text']) > 200 else 'partial'
            rec['match'] = round(score, 2)
    except Exception as e:
        rec['status'] = 'err:' + str(e)[:40]
    items.append(rec)
    time.sleep(0.22)
    if i % 20 == 0:
        print('  进度 %d/%d ｜命中 %d' % (i, len(wx), sum(1 for y in items if y['source'] == 'wechat' and y['status'] == 'ok')))

# ---------- C. 本地副本 ----------
print('[C] 本地副本')
base = os.path.join(W, '资料/IMA知识库同步')
for root, _, fs in os.walk(base):
    for f in fs:
        if f.startswith('.') or not f.lower().endswith(('.pdf', '.md', '.html', '.xlsx', '.docx')):
            continue
        p = os.path.join(root, f)
        text = ''
        try:
            if f.lower().endswith('.pdf'):
                import fitz
                doc = fitz.open(p)
                text = '\n'.join(doc[i].get_text() for i in range(min(6, doc.page_count)))
            elif f.lower().endswith(('.md', '.html')):
                text = open(p, encoding='utf-8', errors='ignore').read()
                text = re.sub(r'<[^>]+>', ' ', text)
            elif f.lower().endswith('.xlsx'):
                import openpyxl
                wb = openpyxl.load_workbook(p, read_only=True, data_only=True)
                buf = []
                for ws in wb.worksheets[:2]:
                    for row in ws.iter_rows(max_row=40, values_only=True):
                        buf.append(' '.join(str(c) for c in row if c is not None))
                text = '\n'.join(buf)
            elif f.lower().endswith('.docx'):
                import docx
                text = '\n'.join(par.text for par in docx.Document(p).paragraphs[:200])
        except Exception as e:
            text = ''
        items.append({'source': 'local', 'title': f, 'path': os.path.relpath(p, base),
                      'date': dt.date.fromtimestamp(os.path.getmtime(p)).isoformat(),
                      'status': 'ok' if len(text) > 200 else 'partial', 'url': '', 'text': text[:6000]})

json.dump({'cutoff': CUTOFF.isoformat(), 'items': items},
          open(os.path.join(W, '数据/ima_corpus.json'), 'w'), ensure_ascii=False, indent=1)
from collections import Counter
print('完成：%d 条' % len(items), Counter((x['source'], x['status']) for x in items).most_common())