# -*- coding: utf-8 -*-
"""客户数据全量拉取（修复版）：session_id 分页，绕开 MCP LIMIT/OFFSET 1000 硬上限。

背景（2026-10-28 实测）：
  pull_customers.py 用 `LIMIT 50 OFFSET n` 单轮翻页，OFFSET 一过 1000 服务端就再不给数据，
  1761 家客户被静默截断成 1050 条 —— 直接覆盖会把 711 家客户从工作台抹掉。
  同一查询的返回体里 list[0].count=1761、带 session_id + group_infos，
  ⇒ 改用 session 分页（group_pagination_list: page_num）可拿全量，分页不受 OFFSET 限制。

用法: /usr/bin/python3 pull_customers_sess.py
输出: mcp_customers.json（格式与旧文件一致：{"updated_at", "customers": [...]}）
安全: ① 拉取条数 < MIN_OK 或 < 旧文件条数×0.95 → 拒绝写盘 exit 1
      ② 写盘前备份旧文件 _bak_customers_pre<YYMMDD_HHMM>.json
"""
import json
import math
import os
import shutil
import subprocess
import sys
import time
from datetime import datetime

BASE = os.path.dirname(os.path.abspath(__file__))
OUT = os.path.join(BASE, 'mcp_customers.json')
TK = 'm-abfb29e8-3104-434f-9944-8d0bb592f8cd'
PK = '6593cd71471290e3cc6be6e6'
COLUMNS = ('name, field_c3224b, field_5ed7ab, field_c8e80d, field_17186c, field_8c870a, '
           'field_79aa0b, field_6415cf, work_item_id, field_e62869, owner, current_status_operator')
MIN_OK = 1600          # 客户总数正常 ≥1760；低于此视为 MCP 大面积失败
DEGRADE = 0.95         # 相对旧文件允许的最大降幅


def mcp(method, args, timeout=60):
    cmd = ['curl', '-s', '--max-time', '55', '-X', 'POST',
           'https://project.feishu.cn/mcp_server/v1',
           '-H', 'X-Mcp-Token: %s' % TK, '-H', 'Content-Type: application/json',
           '-d', json.dumps({"jsonrpc": "2.0", "method": "tools/call",
                             "params": {"name": method, "arguments": args}, "id": 1})]
    out = None
    for _ in range(3):
        try:
            r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
            if r.stdout.strip():
                out = r.stdout
                break
        except subprocess.TimeoutExpired:
            print('     ⏱ 超时重试', file=sys.stderr)
            time.sleep(2)
    if not out:
        return None
    try:
        d = json.loads(out)
    except Exception:
        return None
    for c in (d.get('result') or {}).get('content') or []:
        t = (c.get('text') or '').strip()
        if not t.startswith('{'):
            continue
        try:
            return json.loads(t)
        except Exception:
            continue
    return None


def parse_items(result):
    """与旧版 pull_customers.py 完全一致的扁平行解析，保证下游字段结构不变。"""
    items = []
    if not result:
        return items
    for gid, gitems in (result.get('data') or {}).items():
        for item in (gitems or []):
            fields = {}
            for f in (item.get('moql_field_list') or []):
                k = f['key']
                v = f.get('value')
                if v is None:
                    fields[k] = ''
                elif 'string_value' in v:
                    fields[k] = v['string_value']
                elif 'long_value' in v:
                    fields[k] = v['long_value']
                elif 'user_value' in v:
                    fields[k] = v['user_value']
                elif 'user_value_list' in v:
                    fields[k] = v['user_value_list']
                elif 'key_label_value' in v:
                    fields[k] = v['key_label_value']
                elif 'key_label_value_list' in v:
                    fields[k] = v['key_label_value_list']
                else:
                    fields[k] = ''
            items.append(fields)
    return items


def main():
    t0 = datetime.now()
    print('[%s] 拉取客户数据（session 分页）...' % t0.strftime('%H:%M:%S'))
    mql = "SELECT %s FROM `销售管理`.`客户` ORDER BY name ASC" % COLUMNS
    res = mcp('search_by_mql', {'project_key': PK, 'mql': mql})
    if not res:
        print('🔴 首页无返回，保留旧文件')
        sys.exit(1)
    sid = res.get('session_id')
    lst = (res.get('list') or [{}])[0]
    count = lst.get('count') or 0
    gid = ((lst.get('group_infos') or [{}])[0].get('group_id')) or '1'
    print('  服务端 count=%s ｜ session=%s ｜ group=%s' % (count, (sid or '')[:12], gid))

    seen, items = set(), []

    def absorb(rows):
        n = 0
        for i in rows:
            wid = str(i.get('work_item_id', ''))
            if wid and wid in seen:
                continue
            if wid:
                seen.add(wid)
            items.append(i)
            n += 1
        return n

    absorb(parse_items(res))
    pages = max(1, int(math.ceil(count / 50.0)))

    # 第一策略：session 分页
    if sid:
        for p in range(2, pages + 1):
            r2 = mcp('search_by_mql', {'project_key': PK, 'session_id': sid,
                                       'group_pagination_list': [{'group_id': gid, 'page_num': p}]})
            got = parse_items(r2) if r2 else []
            if not got:
                print('  第 %d 页空 → session 分页收尾（已取 %d）' % (p, len(items)))
                break
            absorb(got)
            if p % 10 == 0 or p == pages:
                print('  第 %d/%d 页 ｜ 累计 %d' % (p, pages, len(items)))
            time.sleep(0.15)

    # 第二策略：session 分页拿不全时，按 start_time 年份/月份分段 OFFSET 补齐（每段 ≤900 行）
    # ⚠️ 客户表没有 created_at 字段（2026-11-12 实测 ErrMetadataError 3003，91 个字段里无此键），
    #    「创建时间」的字段键是 start_time；用错字段会让兜底静默返回 0 行、永远补不齐。
    #    另：单年可能 >1000 行（2024 年 1207 行），必须再按月拆，否则又撞 OFFSET 1000 上限。
    if len(items) < count * 0.98:
        print('  ⚠️ session 分页仅 %d/%d，改按 start_time 年份/月份分段补齐' % (len(items), count))

        def seg_count(cond):
            rs = mcp('search_by_mql', {'project_key': PK,
                                       'mql': 'SELECT name FROM `销售管理`.`客户` WHERE %s '
                                              'ORDER BY name ASC LIMIT 2' % cond})
            n = (((rs or {}).get('list') or [{}])[0]).get('count')
            return n if isinstance(n, int) else 0

        def pull_seg(cond):
            off = 0
            while off <= 1000:
                rs = mcp('search_by_mql', {'project_key': PK,
                                           'mql': 'SELECT %s FROM `销售管理`.`客户` WHERE %s '
                                                  'ORDER BY name ASC LIMIT 50 OFFSET %d' % (COLUMNS, cond, off)})
                rows = parse_items(rs) if rs else []
                if not rows:
                    break
                absorb(rows)
                if len(rows) < 50:
                    break
                off += 50
                time.sleep(0.15)

        for y in range(2018, datetime.now().year + 1):
            ycond = "start_time >= '%d-01-01' AND start_time < '%d-01-01'" % (y, y + 1)
            yc = seg_count(ycond)
            if yc == 0:
                print('  %d 年 ｜ 0 条，跳过' % y)
                continue
            if yc <= 900:
                pull_seg(ycond)
            else:
                print('  %d 年 %d 条 >900，按月拆' % (y, yc))
                for m in range(1, 13):
                    lo = '%d-%02d-01' % (y, m)
                    hi = '%d-%02d-01' % (y, m + 1) if m < 12 else '%d-01-01' % (y + 1)
                    mcond = "start_time >= '%s' AND start_time < '%s'" % (lo, hi)
                    if seg_count(mcond):
                        pull_seg(mcond)
            print('  %d 年 ｜ 累计 %d' % (y, len(items)))

    print('  共 %d 条（服务端 count=%s）' % (len(items), count))

    old_n = 0
    if os.path.exists(OUT):
        try:
            old_n = len((json.load(open(OUT, encoding='utf-8')) or {}).get('customers') or [])
        except Exception:
            old_n = 0

    if len(items) < MIN_OK or (old_n and len(items) < old_n * DEGRADE):
        print('🔴 条数异常（本次 %d ｜ 旧 %d ｜ 门槛 ≥%d 且 ≥旧×%.2f）→ 拒绝写盘，保留旧文件'
              % (len(items), old_n, MIN_OK, DEGRADE))
        sys.exit(1)

    if os.path.exists(OUT):
        bak = os.path.join(BASE, '_bak_customers_pre%s.json' % datetime.now().strftime('%y%m%d_%H%M'))
        shutil.copy2(OUT, bak)
        print('  备份 → %s' % os.path.basename(bak))

    payload = {'updated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
               'customers': items}
    tmp = OUT + '.tmp'
    with open(tmp, 'w', encoding='utf-8') as f:
        json.dump(payload, f, ensure_ascii=False)
    os.replace(tmp, OUT)
    print('✅ 写入 %s（%d 条，旧 %d 条，%.1f KB）'
          % (OUT, len(items), old_n, os.path.getsize(OUT) / 1024.0))


if __name__ == '__main__':
    main()