# -*- coding: utf-8 -*-
"""R1 卡片索引：扫描 知识库-copilot，生成 数据/kb_index.json（供 ai_chat.py 检索层用）
- 实体文件去重（软链接归并到真实文件，路径进 alias）
- 自动抽关键词、摘要、line、owner、时效
- 标出同名重复组（供 R4 合并）
用法：python3 脚本/kb_build_index.py
"""
import os, re, json, hashlib, datetime, collections

ROOT = os.path.expanduser('~/Desktop/Hermes输出-工作类')
KB = os.path.join(ROOT, '知识库-copilot')
TL = os.path.join(KB, '_文本层')
OUT = os.path.join(ROOT, '数据', 'kb_index.json')
READABLE = {'.md', '.pdf'}
LINE_OF_TOP = {'公用': '公用', '诊断原料': 'da', '生命科学': 'ls'}
SKIP_DIRS = {'.git', '_重复归档', '__pycache__'}
GEN_TOKENS = {'知识库', '资料', '文档', '文件', '附件', '汇总', '合集', '版本', '最新', '最终',
              '副本', '模板', '样例', '示例', 'pic', 'thumb', 'hd', 'img', 'image', 'doc', 'docx',
              'xls', 'xlsx', 'ppt', 'pptx', 'pdf', 'md', 'txt', 'zip', 'jpg', 'png', 'xmind',
              '2024', '2025', '2026'}
SUFFIX = re.compile(r'(_1|_副本|_copy|_final|_最终版|_最新版|_thumb|_hd|_thumb_1|_hd_1|\(\d\))+$')


def line_of(relpath):
    return LINE_OF_TOP.get(relpath.split(os.sep)[0], '公用')


def yaml_head(path):
    """读 MD 头部的 YAML（-- 之间的 key: value）+ 首个非空正文行"""
    try:
        with open(path, encoding='utf-8', errors='ignore') as f:
            head = f.read(4000)
    except Exception:
        return {}, ''
    meta = {}
    if head.startswith('---'):
        end = head.find('\n---', 3)
        if end > 0:
            for ln in head[3:end].splitlines():
                if ':' in ln:
                    k, v = ln.split(':', 1)
                    meta[k.strip()] = v.strip()
    body_lines = []
    for ln in head.splitlines():
        s = ln.strip()
        if not s or s.startswith('---') or s.startswith('#') or s.startswith('>') or ':' in s[:14] and s.startswith(tuple(meta.keys()) if meta else ()):
            continue
        body_lines.append(s)
        if len(' '.join(body_lines)) > 200:
            break
    return meta, ' '.join(body_lines)[:200]


def terms_from(*names):
    """从文件名抽关键词：中文 2-6 字块 + 英文/数字 token + SKU 编码"""
    out = []
    for nm in names:
        nm = os.path.splitext(os.path.basename(nm))[0]
        nm = SUFFIX.sub('', nm)
        for m in re.findall(r'[A-Z]{1,6}\d{2,6}(?:-[A-Z0-9]+)?', nm):     # SKU
            out.append(m)
            out.append(m.lower())
        for tok in re.split(r'[^0-9A-Za-z\u4e00-\u9fff]+', nm):
            tok = tok.strip()
            if not tok or tok.lower() in GEN_TOKENS:
                continue
            if re.fullmatch(r'[0-9A-Za-z]+', tok):
                if len(tok) >= 3:
                    out.append(tok)
                    out.append(tok.lower())
            elif 2 <= len(tok) <= 12:
                out.append(tok)
    seen, res = set(), []
    for t in out:
        if t and t not in seen:
            seen.add(t)
            res.append(t)
    return res[:60]


def main():
    entries, by_real = {}, {}
    allp = []
    for r, ds, fs in os.walk(KB):
        ds[:] = [d for d in ds if d not in SKIP_DIRS and not d.startswith('.')]
        for f in fs:
            if f.startswith('.') or f == '.DS_Store':
                continue
            p = os.path.join(r, f)
            if os.path.splitext(f)[1].lower() not in READABLE:
                continue
            allp.append(p)

    for p in allp:
        real = os.path.realpath(p)
        rp = os.path.relpath(p, KB)
        if real in by_real:
            e = by_real[real]
            if rp not in e['paths']:
                e['paths'].append(rp)
            if os.path.islink(p):
                e['symlinks'].append(rp)
            e['keywords'] = list(dict.fromkeys(e['keywords'] + terms_from(rp)))
            continue
        is_sym = os.path.islink(p)
        size = os.path.getsize(real) if os.path.exists(real) else 0
        mt = datetime.datetime.fromtimestamp(os.path.getmtime(real)).strftime('%Y-%m-%d')
        meta, summary = yaml_head(real)
        origin = meta.get('origin', '')
        title = meta.get('title') or SUFFIX.sub('', os.path.splitext(os.path.basename(real))[0])
        line = meta.get('line') or line_of(rp)
        e = {
            'id': hashlib.md5(os.path.relpath(real, KB).encode('utf-8')).hexdigest()[:10],
            'title': title,
            'paths': [rp],
            'symlinks': [rp] if is_sym else [],
            'content_path': os.path.relpath(real, KB),
            'origin': origin,
            'ext': os.path.splitext(real)[1].lower(),
            'in_text_layer': os.path.relpath(real, KB).startswith('_文本层'),
            'line': line,
            'owner': meta.get('owner', '待定'),
            'updated': mt,
            'words': int(meta.get('words') or 0) or (size if os.path.splitext(real)[1].lower() == '.pdf' else 0),
            'confidence': meta.get('confidence', '原文'),
            'auto': meta.get('auto', 'false') == 'true',
            'truncated': meta.get('truncated', 'false') == 'true',
            'summary': summary.replace('|', '/')[:200],
            'keywords': terms_from(rp, origin),
            'dup_key': re.sub(r'[^0-9a-z\u4e00-\u9fff]+', '', SUFFIX.sub('', title.lower())),
        }
        entries[os.path.relpath(real, KB)] = e
        by_real[real] = e

    # 同名重复组
    groups = collections.defaultdict(list)
    for k, e in entries.items():
        groups[e['dup_key']].append(k)
    dups = {g: sorted(v) for g, v in groups.items() if len(v) > 1}
    for g, v in dups.items():
        for k in v:
            entries[k]['dup_group'] = g[:24]

    # 完全同内容条目合并（同文档跨目录拷贝 / _1 副本 / md+md 并存）：保留一条，其余进 alias
    byhash = {}
    for k, e in entries.items():
        fp = os.path.join(KB, e['content_path'])
        try:
            with open(fp, 'rb') as f:
                raw = f.read()
            if e['ext'] == '.md':
                txt = raw.decode('utf-8', 'ignore')
                if txt.startswith('---'):                        # 剥 YAML 头（origin 不同）
                    end = txt.find('\n---', 3)
                    if 0 < end < 4000:
                        txt = txt[end + 4:]
                txt = '\n'.join(ln for ln in txt.splitlines()
                                if '自动转卡（原件：' not in ln)      # 剥来源行
                txt = re.sub(r'([-_ ](副本|复件|copy|hd|thumb|\d{1,2}))+', '', txt)  # 抹副本后缀
                raw = re.sub(r'\s+', ' ', txt).encode('utf-8')    # 空白归一
            h = hashlib.md5(raw).hexdigest()
        except Exception:
            continue
        e['md5'] = h
        byhash.setdefault(h, []).append(k)
    merged = 0
    for h, keys in byhash.items():
        if len(keys) < 2:
            continue
        es = [entries[k] for k in keys]
        keep = min(es, key=lambda e: (e['in_text_layer'], e['line'] != '公用', len(e['paths'][0])))
        for e in es:
            if e is keep or e['id'] not in entries:
                continue
            for p in e['paths']:
                if p not in keep['paths']:
                    keep['paths'].append(p)
            entries.pop(e['id'], None)
            merged += 1
    print('同内容条目合并：%d 条并入主条目（去重后 %d 条）' % (merged, len(entries)))

    # PDF 文本缓存（避免每次问答冷读几百页年报）
    cache_dir = os.path.join(ROOT, '数据', 'kb_text_cache')
    os.makedirs(cache_dir, exist_ok=True)
    ncache = npdf = 0
    for e in entries.values():
        if e['ext'] != '.pdf':
            continue
        npdf += 1
        rel = os.path.join('数据', 'kb_text_cache', e['id'] + '.txt')
        dst = os.path.join(ROOT, rel)
        src = os.path.join(KB, e['content_path'])
        try:
            if (not os.path.exists(dst)) or os.path.getmtime(dst) < os.path.getmtime(src):
                import pymupdf
                txt = '\n'.join(pg.get_text() for pg in pymupdf.open(src))
                with open(dst, 'w', encoding='utf-8') as f:
                    f.write(txt)
                ncache += 1
        except Exception as ex:
            print('  PDF 缓存失败 %s：%s' % (e['content_path'], ex))
            continue
        e['cache'] = rel
    print('PDF 文本缓存：新增/更新 %d（共 %d 个 PDF）' % (ncache, npdf))

    ent = sorted(entries.values(), key=lambda e: (not e['in_text_layer'], e['line'], e['title']))
    byline = collections.Counter()
    byline.update(e['line'] for e in ent)
    idx = {
        'built': datetime.datetime.now().isoformat(timespec='seconds'),
        'kb_dir': '知识库-copilot',
        'counts': {'entries': len(ent), 'dup_groups': len(dups),
                   'auto_cards': sum(1 for e in ent if e['auto']),
                   'in_text_layer': sum(1 for e in ent if e['in_text_layer']),
                   'by_line': dict(byline)},
        'entries': ent,
    }
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    json.dump(idx, open(OUT, 'w', encoding='utf-8'), ensure_ascii=False)
    print('索引条目 %d（转卡层 %d，自动卡 %d）｜同名重复组 %d｜分线 %s'
          % (len(ent), idx['counts']['in_text_layer'], idx['counts']['auto_cards'],
             len(dups), dict(byline)))
    print('输出：数据/kb_index.json （%.1f KB）' % (os.path.getsize(OUT) / 1024))
    kw = collections.Counter()
    for e in ent:
        kw.update(e['keywords'])
    print('高频关键词 top20：', '、'.join(k for k, _ in kw.most_common(20)))


if __name__ == '__main__':
    main()