#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""诊断市场报告数据体检：字段完整性 / URL 可达性 / 重复 / 口径冲突
用法：python verify_diag_data.py
"""
import json
import os
import re
import sys
from collections import Counter

import requests

D = '/Users/liuxinyuan/Desktop/Hermes输出-工作类/数据/diag_market_2025_2027'
FILES = ['market.json', 'companies.json', 'materials.json', 'academic.json']


def load(f):
    p = os.path.join(D, f)
    if not os.path.exists(p):
        return None
    try:
        return json.load(open(p, encoding='utf-8'))
    except Exception as e:
        print('✗ %s JSON 解析失败：%s' % (f, e))
        return None


def walk_urls(o):
    if isinstance(o, dict):
        for k, v in o.items():
            if k in ('url', 'source_url') and isinstance(v, str) and v.startswith('http'):
                yield v
            else:
                yield from walk_urls(v)
    elif isinstance(o, list):
        for x in o:
            yield from walk_urls(x)


def walk_numbers(o):
    """收集带 value 的记录，检查五要素"""
    bad = 0
    total = 0
    if isinstance(o, dict):
        if 'value' in o:
            total += 1
            need = ['unit', 'year', 'source', 'url']
            miss = [k for k in need if not o.get(k)]
            if o.get('value') is None:
                miss.append('value=null')
            if miss:
                bad += 1
        for v in o.values():
            b, t = walk_numbers(v)
            bad += b
            total += t
    elif isinstance(o, list):
        for x in o:
            b, t = walk_numbers(x)
            bad += b
            total += t
    return bad, total


def main():
    report = []
    for f in FILES:
        d = load(f)
        if d is None:
            report.append((f, '缺失', 0, 0, 0))
            continue
        urls = list(walk_urls(d))
        bad_num, total_num = walk_numbers(d)
        gaps = len(d.get('gaps') or [])
        report.append((f, '可读', len(urls), total_num, bad_num, gaps))
        # 重复 URL
        dup = [u for u, c in Counter(urls).items() if c > 1]
        if dup:
            print('  %s 重复 URL %d 个（示例：%s）' % (f, len(dup), dup[0][:70]))
    print('\n文件 | 状态 | URL 数 | 数字条数 | 缺要素 | gaps')
    for r in report:
        print('  %-16s %-4s %6s %8s %7s %5s' % r)

    # 抽样可达性（每文件最多 8 条）
    print('\n抽样 URL 可达性：')
    for f in FILES:
        d = load(f)
        if not d:
            continue
        urls = list(dict.fromkeys(walk_urls(d)))[:8]
        ok = 0
        for u in urls:
            try:
                r = requests.head(u, timeout=8, allow_redirects=True,
                                  headers={'User-Agent': 'Mozilla/5.0'})
                ok += 1 if r.status_code < 400 else 0
            except Exception:
                pass
        if urls:
            print('  %-16s %d/%d 可达' % (f, ok, len(urls)))


if __name__ == '__main__':
    main()