#!/usr/bin/env python3
"""Step 2 of the ai-industry-2026 post ("Which chatbot does each industry depend on?"): from the website-traffic rows
(out/ai-industry/raw.json, build 0.1.12 with aiChatbotShares) compute every figure the post quotes, by industry.

Writes:
  out/ai-industry/sample.json   one compact row per site in the sample (domain, industry, visits, AI referrals, shares)
  out/ai-industry/results.json  every aggregate the post uses
  out/ai-industry/tables.md     the same figures as Markdown, for checking by eye (also printed)

Definitions (Similarweb estimates for the latest published month, as returned by the Actor):
  AI referrals       aiChatbotVisits, Similarweb's AI-referral figure for the site (the shares come from the same data)
  chatbot share      aiChatbotShares[bot], the bot's share of the site's AI referrals, in percent. Similarweb gives it
                     for its top few chatbots per site (usually 3); a bot not listed counts as 0, so every share is a
                     floor and the shares need not add up to 100.
  sample             Tranco top 5,000 (infrastructure hosts removed) with a chatbot split and at least MIN_AI_REFERRALS
                     AI referrals in the month; AI companies' own sites are left out (a chatbot linking to its maker)
  average share      unweighted mean over the sites in a group: each site counts once, big or small
  pooled share       sum(AI referrals x share) / sum(AI referrals): the bot's share of all the group's AI referrals
  leads              the bot with the largest share on that site
  index              group's average share for a bot / the whole sample's average share for that bot (1.0 = typical)
Run: python3 distribution/analysis/ai-industry-2026-analyse.py
"""
import json
import statistics as st
from collections import Counter, defaultdict
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = HERE / 'out' / 'ai-industry'
MIN_AI_REFERRALS = 5_000      # below this, a site's chatbot split rests on a handful of clicks
MIN_SITES = 15                # industries with fewer sites are folded into "Everything else"
SMALL = 30                    # industries with fewer sites than this are flagged as small samples
MIN_AI_FOR_LISTS = 20_000    # site lists (most Claude-dependent etc.) only include sites with this many AI referrals
LIST_SKIP = {'Adult'}         # adult sites count in every table but are not named in the site lists
BOTS = ['chatgpt', 'gemini', 'perplexity', 'claude', 'copilot', 'grok', 'deepseek']
NAME = {'chatgpt': 'ChatGPT', 'gemini': 'Gemini', 'perplexity': 'Perplexity', 'claude': 'Claude', 'copilot': 'Copilot',
        'grok': 'Grok', 'deepseek': 'DeepSeek'}
AI_OWN = {'openai.com', 'chatgpt.com', 'anthropic.com', 'claude.ai', 'perplexity.ai', 'deepseek.com', 'huggingface.co',
          'cursor.sh', 'chatgpt.site', 'gemini.google.com', 'grok.com', 'x.ai', 'mistral.ai', 'character.ai', 'poe.com',
          'claude.com', 'claudeusercontent.com', 'doubao.com', 'flowgpt.com', 'higgsfield.ai'}

# Our industry groups, built from Similarweb's own categories ("Top > Sub"). A sub-category rule wins over a top-level one.
SUB = {
    'Programming and Developer Software': 'Software and developer tools',
    'Web Hosting and Domain Names': 'Software and developer tools',
    'Computer Security': 'Software and developer tools',
    'Graphics Multimedia and Web Design': 'Software and developer tools',
    'File Sharing and Hosting': 'Software and developer tools',
    'Search Engines': 'Search engines and portals',
    'Social Networks and Online Communities': 'Social networks and forums',
    'Consumer Electronics': 'Telecoms and electronics',
    'Computer Hardware': 'Telecoms and electronics',
    'Telecommunications': 'Telecoms and electronics',
    'Universities and Colleges': 'Universities and education',
    'Education': 'Universities and education',
    'Business Training': 'Universities and education',
    'Real Estate': 'Real estate',
    'Jobs and Employment': 'Jobs and careers',
    'Human Resources': 'Jobs and careers',
}
TOP = {
    'Computers Electronics and Technology': 'Other tech and web services',
    'Business and Consumer Services': 'Business services and marketing',
    'E-Commerce and Shopping': 'Shopping and e-commerce',
    'Lifestyle': 'Shopping and e-commerce',
    'News and Media': 'News and media',
    'Finance': 'Finance and crypto',
    'Crypto and Blockchain': 'Finance and crypto',
    'Law and Government': 'Government and law',
    'Science and Education': 'Science and research',
    'Reference Materials': 'Reference',
    'Travel and Tourism': 'Travel',
    'Health': 'Health',
    'Arts and Entertainment': 'Entertainment and streaming',
    'Games': 'Games',
    'Sports': 'Sports and betting',
    'Gambling': 'Sports and betting',
    'Food and Drink': 'Food and drink',
    'Adult': 'Adult',
    'Vehicles': 'Vehicles',
    'Jobs and Career': 'Jobs and careers',
}


def industry(cat):
    if not cat or cat == 'None':
        return 'Uncategorised'
    top, _, sub = cat.partition(' > ')
    return SUB.get(sub) or TOP.get(top) or 'Everything else'


cand = json.loads((OUT / 'candidates.json').read_text())
raw = json.loads((OUT / 'raw.json').read_text())
rank_of = {c['domain']: c['rank'] for c in cand['domains']}
items = raw['items']
with_est = [r for r in items if r.get('trafficAvailable') is True]
with_split = [r for r in with_est if r.get('aiChatbotShares')]
rows, ai_own, too_small = [], [], 0
for r in with_split:
    if r['domain'] in AI_OWN or (r.get('category') or '').startswith('Ai Chatbots and Tools'):
        ai_own.append(r['domain'])
        continue
    if (r.get('aiChatbotVisits') or 0) < MIN_AI_REFERRALS:
        too_small += 1
        continue
    sh = r['aiChatbotShares']
    rows.append({'rank': rank_of.get(r['domain']), 'domain': r['domain'], 'category': r.get('category'),
                 'industry': industry(r.get('category')), 'visits': r.get('monthlyVisits'),
                 'ai': r['aiChatbotVisits'], 'shares': {b: sh.get(b, 0) for b in BOTS},
                 'listed': len(sh), 'lead': max(sh, key=sh.get)})
rows.sort(key=lambda r: r['rank'] or 10**9)
months = Counter(r.get('trafficMonth') for r in with_split)

# Fold small industries into "Everything else"
size = Counter(r['industry'] for r in rows)
for r in rows:
    if size[r['industry']] < MIN_SITES and r['industry'] != 'Uncategorised':
        r['industry'] = 'Everything else'

pct = lambda a, b: round(100 * a / b, 1) if b else None  # noqa: E731
mean = lambda xs: round(st.mean(xs), 1) if xs else None  # noqa: E731


def summarise(rs):
    ai = sum(r['ai'] for r in rs)
    lead = Counter(r['lead'] for r in rs)
    return {
        'sites': len(rs), 'aiReferrals': ai, '_mean': {b: st.mean([r['shares'][b] for r in rs]) for b in BOTS},
        'avgShare': {b: mean([r['shares'][b] for r in rs]) for b in BOTS},
        'medianShare': {b: round(st.median([r['shares'][b] for r in rs]), 1) for b in BOTS},
        'pooledShare': {b: pct(sum(r['ai'] * r['shares'][b] / 100 for r in rs), ai) for b in BOTS},
        'leads': {b: lead[b] for b in BOTS if lead[b]},
        'chatgptLeadsPct': pct(lead['chatgpt'], len(rs)),
        'notChatgptLead': len(rs) - lead['chatgpt'],
        'listedPct': {b: pct(sum(1 for r in rs if r['shares'][b] > 0), len(rs)) for b in BOTS},
        'chatgptUnder50Pct': pct(sum(1 for r in rs if r['shares']['chatgpt'] < 50), len(rs)),
    }


overall = summarise(rows)
groups = defaultdict(list)
for r in rows:
    groups[r['industry']].append(r)
inds = []
for name, rs in groups.items():
    s = summarise(rs)
    s['industry'] = name
    s['small'] = len(rs) < SMALL
    s['index'] = {b: round(s['_mean'][b] / overall['_mean'][b], 2) if overall['_mean'][b] else None for b in BOTS}
    s['secondBot'] = max((b for b in BOTS if b != 'chatgpt'), key=lambda b: s['avgShare'][b])
    s['categories'] = Counter(r['category'] for r in rs).most_common(4)
    inds.append(s)
named = [s for s in inds if s['industry'] not in ('Everything else', 'Uncategorised')]
named.sort(key=lambda s: s['avgShare']['chatgpt'])
others = [s for s in inds if s['industry'] in ('Everything else', 'Uncategorised')]

# Similarweb sub-categories with at least MIN_SUB sites: where each challenger's average share is highest
MIN_SUB = 10
subs = defaultdict(list)
for r in rows:
    if r['industry'] != 'Adult':
        subs[r['category']].append(r)
sub_stats = [{'category': c, 'sites': len(rs), **{b: round(st.mean([r['shares'][b] for r in rs]), 1) for b in BOTS},
              'leads': dict(Counter(r['lead'] for r in rs))} for c, rs in subs.items() if c and len(rs) >= MIN_SUB]
sub_top = {b: sorted(sub_stats, key=lambda x: -x[b])[:6] for b in ['gemini', 'perplexity', 'claude', 'chatgpt']}
sub_low_chatgpt = sorted(sub_stats, key=lambda x: x['chatgpt'])[:6]
# Russian-language web: Perplexity's average share at .ru sites vs the rest
ru = [r for r in rows if r['domain'].endswith('.ru')]
rest = [r for r in rows if not r['domain'].endswith('.ru')]
ru_stats = {'sites': len(ru), 'perplexityAvg': round(st.mean([r['shares']['perplexity'] for r in ru]), 1),
            'perplexityAvgRest': round(st.mean([r['shares']['perplexity'] for r in rest]), 1),
            'perplexityLeads': sum(1 for r in ru if r['lead'] == 'perplexity'),
            'perplexityLeadsAll': sum(1 for r in rows if r['lead'] == 'perplexity')}

# Which bot is second (after ChatGPT) in each named industry
second = Counter(s['secondBot'] for s in named)
not_gemini_second = [s['industry'] for s in named if s['secondBot'] != 'gemini']

# Where each challenger over-indexes most (named industries, not flagged small) and least
solid = [s for s in named if not s['small']]
over = {b: sorted(({'industry': s['industry'], 'sites': s['sites'], 'avgShare': s['avgShare'][b], 'index': s['index'][b]}
                   for s in named), key=lambda x: -x['index']) for b in BOTS}


def site(r):
    return {'rank': r['rank'], 'domain': r['domain'], 'industry': r['industry'], 'category': r['category'],
            'visits': r['visits'], 'ai': r['ai'], 'shares': {b: v for b, v in r['shares'].items() if v}, 'lead': NAME[r['lead']]}


big = [r for r in rows if r['ai'] >= MIN_AI_FOR_LISTS and r['industry'] not in LIST_SKIP]
top_dependent = {b: [site(r) for r in sorted(big, key=lambda r: (-r['shares'][b], -r['ai']))[:15] if r['shares'][b] > 0]
                 for b in ['claude', 'perplexity', 'gemini', 'grok', 'copilot', 'deepseek']}
leaders = {b: sorted((site(r) for r in rows if r['lead'] == b and r['industry'] not in LIST_SKIP), key=lambda s: -s['ai']) for b in BOTS if b != 'chatgpt'}
lead_by_industry = {b: Counter(r['industry'] for r in rows if r['lead'] == b).most_common() for b in BOTS if b != 'chatgpt'}
# Claude ahead of Gemini / Perplexity ahead of Gemini, site by site, by industry
def ahead(a, b, rs):
    return sum(1 for r in rs if r['shares'][a] > r['shares'][b])
pairs = {s['industry']: {'claudeOverGemini': ahead('claude', 'gemini', groups[s['industry']]),
                         'perplexityOverGemini': ahead('perplexity', 'gemini', groups[s['industry']]),
                         'claudeOverPerplexity': ahead('claude', 'perplexity', groups[s['industry']])} for s in inds}
pairs['ALL'] = {'claudeOverGemini': ahead('claude', 'gemini', rows), 'perplexityOverGemini': ahead('perplexity', 'gemini', rows),
                'claudeOverPerplexity': ahead('claude', 'perplexity', rows)}
# The most ChatGPT-reliant industries: share of sites where ChatGPT has >= 80% of AI referrals
for s in inds:
    rs = groups[s['industry']]
    s['chatgpt80Pct'] = pct(sum(1 for r in rs if r['shares']['chatgpt'] >= 80), len(rs))

for s in [overall, *inds]:
    s.pop('_mean')
res = {
    'source': {'tranco': cand['listId'], 'actor': raw['actor'], 'build': raw['build'], 'runs': raw['runs'],
               'apifyUsageUsd': round(raw['usageTotalUsd'], 3), 'trafficMonth': months.most_common(1)[0][0],
               'data': 'Similarweb estimates'},
    'sample': {'candidates': len(cand['domains']), 'excludedInfra': len(cand['excluded']), 'returned': len(items),
               'withEstimate': len(with_est), 'withChatbotSplit': len(with_split), 'aiOwnLeftOut': sorted(ai_own),
               'belowMinAi': too_small, 'minAiReferrals': MIN_AI_REFERRALS, 'sites': len(rows),
               'lastTrancoRank': cand['domains'][-1]['rank'], 'listedCount': dict(Counter(r['listed'] for r in rows)),
               'sharesSumMedian': round(st.median([sum(r['shares'].values()) for r in rows]), 1),
               'minAiForLists': MIN_AI_FOR_LISTS, 'bigSites': len(big), 'minSites': MIN_SITES, 'smallFlag': SMALL},
    'overall': overall,
    'industries': named, 'otherGroups': others,
    'subStats': sorted(sub_stats, key=lambda x: -x['sites']), 'subTop': sub_top, 'subLowChatgpt': sub_low_chatgpt, 'minSubSites': MIN_SUB, 'ru': ru_stats,
    'overIndex': over, 'pairs': pairs, 'namedIndustries': len(named),
    'secondBotCount': dict(second), 'notGeminiSecond': not_gemini_second,
    'topDependent': top_dependent, 'leaders': leaders, 'leadByIndustry': lead_by_industry,
}
(OUT / 'results.json').write_text(json.dumps(res, indent=1, ensure_ascii=False))
(OUT / 'sample.json').write_text(json.dumps({'trafficMonth': res['source']['trafficMonth'], 'tranco': cand['listId'],
                                             'rows': [{k: r[k] for k in ('rank', 'domain', 'category', 'industry', 'visits', 'ai', 'shares')} for r in rows]},
                                            separators=(',', ':')))

# ---------------------------------------------------------------- tables.md (for checking by eye)
f = lambda n: f'{n:,.0f}'  # noqa: E731
md = [f"# ai-industry-2026 (Tranco {cand['listId']}, Similarweb month {res['source']['trafficMonth']}, usage ${res['source']['apifyUsageUsd']})", '',
      json.dumps(res['sample']), '', '## Overall', json.dumps(overall), '', '## Industries (sorted by ChatGPT average share)',
      '| Industry | Sites | AI referrals | ChatGPT avg | Gemini avg | Perplexity avg | Claude avg | Grok avg | Copilot avg | DeepSeek avg | ChatGPT leads % | not ChatGPT | CG>=80 % | 2nd | pooled CG/Gem/Ppx/Cl |',
      '|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|']
for s in named + others:
    a = s['avgShare']
    md.append(f"| {s['industry']}{' (small)' if s.get('small') else ''} | {s['sites']} | {f(s['aiReferrals'])} | {a['chatgpt']} | {a['gemini']} | {a['perplexity']} | {a['claude']} | {a['grok']} | {a['copilot']} | {a['deepseek']} | {s['chatgptLeadsPct']} | {s['notChatgptLead']} | {s.get('chatgpt80Pct')} | {NAME[s['secondBot']]} | {s['pooledShare']['chatgpt']}/{s['pooledShare']['gemini']}/{s['pooledShare']['perplexity']}/{s['pooledShare']['claude']} |")
md += ['', '## Over-index (avg share / overall avg share)']
for b in ['gemini', 'perplexity', 'claude', 'grok', 'copilot']:
    md.append(f"- {NAME[b]}: " + '; '.join(f"{x['industry']} {x['avgShare']} ({x['index']}x, n={x['sites']})" for x in over[b][:5])
              + ' ... lowest: ' + '; '.join(f"{x['industry']} {x['avgShare']} ({x['index']}x)" for x in over[b][-3:]))
md += ['', '## Pairs', json.dumps(pairs)]
for b, lst in top_dependent.items():
    md += ['', f'## Most {NAME[b]}-dependent (>= {MIN_AI_FOR_LISTS:,} AI referrals, not adult)', *[f"- {s['domain']} ({s['industry']}; {f(s['visits'])} visits, {f(s['ai'])} AI) {s['shares']}" for s in lst]]
for b, lst in leaders.items():
    md += ['', f'## {NAME[b]} leads at {overall["leads"].get(b, 0)} sites ({len(lst)} named, adult left out); by industry {lead_by_industry[b]}', ', '.join(f"{s['domain']} ({f(s['ai'])})" for s in lst[:25])]
md += ['', f'## Sub-categories (>= {MIN_SUB} sites, adult left out): highest average share']
for b, lst in sub_top.items():
    md.append(f"- {NAME[b]}: " + '; '.join(f"{x['category']} {x[b]} (n={x['sites']}, leads {x['leads']})" for x in lst))
md.append('- lowest ChatGPT: ' + '; '.join(f"{x['category']} {x['chatgpt']} (n={x['sites']})" for x in sub_low_chatgpt))
md += ['', '## .ru', json.dumps(ru_stats)]
(OUT / 'tables.md').write_text('\n'.join(md) + '\n')
print('\n'.join(md))