#!/usr/bin/env python3
"""Signal recipe 3 (review signal): numbers for the article, from out/recipe-3/reviews.json (recipe-3-run.mjs).
Complaint themes are plain keyword groups matched on each review's title and text, ignoring case; a review can fall in
several themes or none. No model, no sentiment scoring: the star rating is the reviewer's own.
Writes out/recipe-3/results.json (every figure the article quotes) and out/recipe-3/companies.csv (one row per
supplier: TrustScore, star mix of recent reviews, complaint themes, reply rate and speed). Company-level only: no
reviewer names (the run script never stores them), no review text in any output.
Run: python3 distribution/analysis/recipe-3-analysis.py
"""
import csv, json, re, statistics
from collections import Counter, defaultdict
from datetime import datetime, timedelta
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = HERE / 'out' / 'recipe-3'
d = json.loads((OUT / 'reviews.json').read_text())
runs = json.loads((OUT / 'runs.json').read_text())
CO = {c['domain']: c for c in d['companies']}
snap = [r for r in d['snapshot'] if r.get('type') == 'review']
biz = {r['input']: r for r in d['snapshot'] if r.get('type') == 'business'}
comp = [r for r in d['complaints'] if r.get('type') == 'review']
assert not any('reviewerName' in r for r in d['snapshot'] + d['complaints']), 'reviewer names must not be stored'

ts = lambda s: datetime.fromisoformat(s.replace('Z', '+00:00'))
SNAP_AT = ts(runs[1]['startedAt'])  # the complaints run: reply ages are measured from when it read the reviews

THEMES = {
    'Getting through to someone': r'on hold|hold for|wait(?:ing)? times?|call ?backs?|no (?:response|reply|answer)|not (?:responded|replied)|get through|got through|hung up|hang up|cut off|disconnected|no one|nobody|chase|chasing|ignored|phone calls|emails?\b|live chat|web ?chat|\bchat\b',
    'Billing and estimated bills': r'\bbills?\b|billing|billed|overcharg|over-charg|estimated|estimates?\b|statement|back[- ]?bill|charged|charges',
    'Smart meters and readings': r'smart meter|\bmeters?\b|readings?\b|\bIHD\b|in-home display|display unit',
    'Engineers and appointments': r'engineers?|appointments?|install(?:ation|ed|er)?\b|no[- ]show|time slot|visit',
    'Direct debit and payments': r'direct debit|\bDD\b|payment plan|monthly payments?|payments? (?:went|gone|increased|taken)|took .{0,20}payment',
    'Refunds and credit balances': r'refund|credit balance|money back|owed|owe me|in credit',
    'Switching, moving and closing accounts': r'switch(?:ed|ing)?\b|moving (?:home|house|out)|moved (?:home|house|out|in)|new supplier|final bill|clos(?:e|ed|ing) (?:my|the|our) account|cancel',
    'Tariffs and prices': r'tariff|unit rate|standing charge|price|expensive|\brates?\b',
    'Debt collection and court': r'debt|collectors?|collection agenc|bailiff|court|warrant|\bCCJ\b|credit (?:file|rating|score)',
    'Contracts and renewals': r'contract|renewal|renewed|rolled over|rollover|out[- ]of[- ]contract|deemed|broker|termination',
    'Rude or unhelpful staff': r'\brude\b|unhelpful|attitude|incompetent|clueless|no idea|lied|liars?\b|patroni[sz]',
    'Formal complaints and the ombudsman': r'ombudsman|formal complaint|complaints? (?:team|department|process)|escalat',
    'Prepayment and top-ups': r'prepay|pre-pay|top[- ]?ups?|pay as you go|\bPAYG\b|emergency credit',
    'Solar, EV and export': r'solar|export|\bEV\b|electric (?:car|vehicle)|heat pump|batter(?:y|ies)|\bSEG\b',
    'Broadband, phone and other services': r'broadband|wi-?fi|internet|home phone|landline|mobile|\bsim\b|boiler|home ?care|insurance',
}
THEME_RE = {k: re.compile(v, re.I) for k, v in THEMES.items()}
def themes(r):
    t = f"{r.get('title') or ''}\n{r.get('text') or ''}"
    return [k for k, rx in THEME_RE.items() if rx.search(t)]

invited = lambda r: r.get('reviewSource') != 'Organic'
low = lambda r: r['rating'] <= 2

# Snapshot: star mix of every recent review, invited against organic.
by_co = defaultdict(list)
for r in snap:
    by_co[r['input']].append(r)
star_mix = Counter(r['rating'] for r in snap)
org = [r for r in snap if not invited(r)]
inv = [r for r in snap if invited(r)]
src = Counter(r['reviewSource'] for r in snap)

# Complaints: 1-2 star reviews from the last 30 days.
comp_by_co = defaultdict(list)
for r in comp:
    r['_themes'] = themes(r)
    comp_by_co[r['input']].append(r)
theme_n = Counter(t for r in comp for t in r['_themes'])
no_theme = sum(1 for r in comp if not r['_themes'])

def reply_stats(rows):
    """Replies to 1-2 star reviews. A review read minutes after it was posted hasn't had a fair chance of a reply yet,
    so each rate only counts reviews old enough: 'within 24 hours' on reviews at least 24 hours old when read, 'at all'
    on reviews at least 72 hours old. The median is over every reply we saw."""
    age = lambda r: (SNAP_AT - ts(r['publishedAt'])).total_seconds() / 3600
    hours = lambda r: (ts(r['replyPublishedAt']) - ts(r['publishedAt'])).total_seconds() / 3600
    d1 = [r for r in rows if age(r) >= 24]
    d3 = [r for r in rows if age(r) >= 72]
    fast = [r for r in d1 if r.get('replyPublishedAt') and hours(r) <= 24]
    rep3 = [r for r in d3 if r.get('replyPublishedAt')]
    hrs = [hours(r) for r in rows if r.get('replyPublishedAt')]
    return {
        'reviews': len(rows), 'atLeast24h': len(d1), 'within24h': len(fast),
        'within24hRate': round(len(fast) / len(d1), 3) if d1 else None,
        'atLeast72h': len(d3), 'replied72h': len(rep3), 'replyRate': round(len(rep3) / len(d3), 3) if d3 else None,
        'repliedAny': len(hrs), 'medianHoursToReply': round(statistics.median(hrs), 1) if hrs else None,
    }

WEEK_AGO = SNAP_AT - timedelta(days=7)
rows = []
for dom, c in CO.items():
    s, cp, b = by_co.get(dom, []), comp_by_co.get(dom, []), biz.get(dom, {})
    tn = Counter(t for r in cp for t in r['_themes'])
    rs = reply_stats(cp)
    rows.append({
        'company': c['name'], 'domain': dom, 'segment': c['segment'],
        'trustScore': b.get('trustScore'), 'reviewsAllTime': b.get('reviewCount'), 'reviewsLast12Months': b.get('reviewsLast12Months'),
        'recentReviews': len(s), 'recentFrom': min((r['publishedAt'][:10] for r in s), default=''), 'recentTo': max((r['publishedAt'][:10] for r in s), default=''),
        'recentLowShare': round(sum(low(r) for r in s) / len(s), 3) if s else None,
        'recentInvitedShare': round(sum(invited(r) for r in s) / len(s), 3) if s else None,
        'organicLowShare': round(sum(low(r) for r in s if not invited(r)) / max(1, sum(not invited(r) for r in s)), 3) if any(not invited(r) for r in s) else None,
        'organicReviews': sum(not invited(r) for r in s),
        'lowReviews30d': len(cp), 'lowReviews30dFrom': min((r['publishedAt'][:10] for r in cp), default=''),
        'topThemes': [t for t, _ in tn.most_common(3)], 'themeCounts': dict(tn),
        # 1-2 star reviews in the 7 days before the run. 'capped' means the 100-review limit was reached inside those
        # 7 days, so the true number is higher.
        'lowLast7Days': sum(ts(r['publishedAt']) >= WEEK_AGO for r in cp),
        'lowLast7DaysCapped': len(cp) >= 100 and min(ts(r['publishedAt']) for r in cp) >= WEEK_AGO,
        # Average 1-2 star reviews a day: over 30 days, or over the days the 100 we read cover when the limit was hit.
        'lowPerDay': round(len(cp) / (((SNAP_AT - min(ts(r['publishedAt']) for r in cp)).total_seconds() / 86400) if len(cp) >= 100 else 30), 1) if cp else 0,
        **{f'reply_{k}': v for k, v in rs.items()},
    })

with (OUT / 'companies.csv').open('w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['company', 'domain', 'segment', 'trustscore', 'reviews_all_time', 'reviews_last_12_months',
                'recent_reviews_read', 'recent_from', 'recent_to', 'recent_1_2_star_share', 'recent_invited_share',
                'organic_reviews', 'organic_1_2_star_share', 'low_reviews_last_30_days_read', 'top_complaint_themes',
                'low_reviews_last_7_days', 'last_7_days_is_a_minimum', 'low_reviews_per_day', 'low_reviews_24h_or_older', 'replied_within_24h_share',
                'low_reviews_72h_or_older', 'replied_share', 'median_hours_to_reply'])
    for r in sorted(rows, key=lambda r: -(r['recentLowShare'] or 0)):
        w.writerow([r['company'], r['domain'], r['segment'], r['trustScore'], r['reviewsAllTime'], r['reviewsLast12Months'],
                    r['recentReviews'], r['recentFrom'], r['recentTo'], r['recentLowShare'], r['recentInvitedShare'],
                    r['organicReviews'], r['organicLowShare'], r['lowReviews30d'], '; '.join(r['topThemes']),
                    r['lowLast7Days'], r['lowLast7DaysCapped'], r['lowPerDay'], r['reply_atLeast24h'], r['reply_within24hRate'],
                    r['reply_atLeast72h'], r['reply_replyRate'], r['reply_medianHoursToReply']])

seg = {}
for sname in ['household', 'Northern Ireland', 'business']:
    ss = [r for r in snap if CO[r['input']]['segment'] == sname]
    seg[sname] = {'companies': sum(1 for c in CO.values() if c['segment'] == sname), 'reviews': len(ss), 'lowShare': round(sum(low(r) for r in ss) / len(ss), 3)}

all_reply = reply_stats(comp)
snap_low_rep = reply_stats([r for r in snap if low(r)])
snap_high_rep = reply_stats([r for r in snap if r['rating'] >= 4])
res = {
    'snapshot': runs[0]['startedAt'][:10],
    'runs': [{k: r[k] for k in ('label', 'runId', 'status', 'seconds', 'usageTotalUsd', 'chargedEventCounts', 'itemCount')} for r in runs],
    'companies': len(CO),
    'snapshotReviews': len(snap), 'snapshotStars': dict(sorted(star_mix.items())),
    'snapshotLowShare': round(sum(low(r) for r in snap) / len(snap), 3),
    'invitedReviews': len(inv), 'organicReviews': len(org), 'sources': dict(src),
    'invitedLowShare': round(sum(low(r) for r in inv) / len(inv), 3), 'organicLowShare': round(sum(low(r) for r in org) / len(org), 3),
    'invitedFiveShare': round(sum(r['rating'] == 5 for r in inv) / len(inv), 3), 'organicFiveShare': round(sum(r['rating'] == 5 for r in org) / len(org), 3),
    'segments': seg,
    'complaints': len(comp), 'complaintsCompanies': sum(1 for v in comp_by_co.values() if v),
    'themes': [{'theme': t, 'reviews': n, 'share': round(n / len(comp), 3)} for t, n in theme_n.most_common()],
    'noTheme': no_theme,
    'replyComplaints': all_reply, 'replySnapshotLow': snap_low_rep, 'replySnapshotHigh': snap_high_rep,
    'perCompany': rows,
    'priceStarterUsd': {r['label']: round(sum(v for k, v in (r['chargedEventCounts'] or {}).items() if k != 'apify-actor-start') * 0.0005 + 0.00005, 4) for r in runs},
}
(OUT / 'results.json').write_text(json.dumps(res, indent=1, ensure_ascii=False))
print(json.dumps({k: v for k, v in res.items() if k != 'perCompany'}, indent=1, ensure_ascii=False))
for r in sorted(rows, key=lambda r: -(r['recentLowShare'] or 0)):
    print(f"{r['company']:26} {r['segment'][:5]:5} TS={r['trustScore']} n={r['recentReviews']:3} low={r['recentLowShare']} inv={r['recentInvitedShare']} orgN={r['organicReviews']:3} orgLow={r['organicLowShare']} | 30d low={r['lowReviews30d']:3} from {r['lowReviews30dFrom']} wk={r['lowLast7Days']}{'+' if r['lowLast7DaysCapped'] else ''} /day={r['lowPerDay']} 24h={r['reply_within24h']}/{r['reply_atLeast24h']} all={r['reply_replied72h']}/{r['reply_atLeast72h']} med={r['reply_medianHoursToReply']}h | {r['topThemes']}")