#!/usr/bin/env python3
"""Signal recipe 2 (tech-stack signal): numbers for the article, from out/recipe-2/domains.json (recipe-2-run.mjs)
and out/recipe-2/shopify-candidates.json (recipe-2-list.mjs).
Writes out/recipe-2/results.json (every figure the article quotes), out/recipe-2/stores.csv (one row per store, the
flat sheet the n8n workflow would build) and out/recipe-2/segment-*.csv (the domain lists for each segment).
Company-level data only: domain-intel collects no registrant or personal data.
Run: python3 distribution/analysis/recipe-2-analysis.py
"""
import csv, json
from collections import Counter
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = HERE / 'out' / 'recipe-2'
items = json.loads((OUT / 'domains.json').read_text())['items']
run = json.loads((OUT / 'run.json').read_text())
summary = json.loads((OUT / 'summary.json').read_text())
cand = json.loads((OUT / 'shopify-candidates.json').read_text())
rank = {m['domain']: m['rank'] for m in cand['sample']}


def cat(d, c):
    return (d.get('techCategories') or {}).get(c) or []


def names(d):
    return {t['name'] for t in d.get('techStack') or []}


seen = [d for d in items if not (d.get('errors') or {}).get('tech')]  # homepage analysed
N = len(seen)
EMAIL_TOOLS = {'Klaviyo', 'MailChimp', 'Sendinblue', 'Omnisend', 'Privy', 'Ometria', 'MoEngage', 'HubSpot', 'Braze', 'Salesforce Marketing Cloud Account Engagement', 'Connectif', 'WebEngage', 'Netcore Cloud', 'Automizely', 'Attentive', 'Postscript', 'Insider', 'Bloomreach', 'Dotdigital', 'Iterable', 'Emarsys'}
SUPPORT = {'Gorgias', 'Zendesk', 'Zendesk Chat', 'Re:amaze', 'Kustomer', 'Freshchat', 'Freshdesk', 'Ada', 'Intercom', 'Tidio', 'LiveChat', 'Crisp Live Chat', 'Help Scout', 'Richpanel', 'eDesk', 'Oct8ne', 'Zendesk Sunshine Conversations'}

def row(d):
    n = names(d)
    return {
        'domain': d['domain'], 'tranco_rank': rank.get(d['domain']), 'site_title': (d.get('title') or '')[:80],
        'email_marketing': ', '.join(sorted(n & EMAIL_TOOLS)), 'reviews': ', '.join(sorted(cat(d, 'Reviews'))),
        'support': ', '.join(sorted(n & SUPPORT)), 'bnpl': ', '.join(sorted(cat(d, 'Buy now pay later'))),
        'consent': ', '.join(sorted(cat(d, 'Cookie compliance'))), 'email_host': d.get('emailProvider') or '',
        'dmarc': d.get('dmarcPolicy') or 'missing', 'cdn': d.get('cdn') or '',
    }

rows = sorted((row(d) for d in items), key=lambda r: r['tranco_rank'] or 10**9)
with (OUT / 'stores.csv').open('w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=list(rows[0]))
    w.writeheader(); w.writerows(rows)

klaviyo = [d for d in seen if 'Klaviyo' in names(d)]
segments = {
    'klaviyo-no-reviews': ('Klaviyo, no review app on the homepage', [d for d in klaviyo if not cat(d, 'Reviews')]),
    'no-email-platform': ('No email or SMS marketing tool on the homepage', [d for d in seen if not (names(d) & EMAIL_TOOLS)]),
    'mailchimp-not-klaviyo': ('Mailchimp, not Klaviyo', [d for d in seen if 'MailChimp' in names(d) and 'Klaviyo' not in names(d)]),
    'klaviyo-no-dmarc-enforcement': ('Klaviyo, and DMARC missing or at p=none', [d for d in klaviyo if (d.get('dmarcPolicy') or 'missing') in ('missing', 'none')]),
    'klaviyo-no-dmarc': ('Klaviyo, and no DMARC record at all', [d for d in klaviyo if not d.get('dmarcPolicy')]),
    'reviews-no-email-platform': ('A review app, but no email marketing tool', [d for d in seen if cat(d, 'Reviews') and not (names(d) & EMAIL_TOOLS)]),
    'no-bnpl': ('No buy-now-pay-later option on the homepage', [d for d in seen if not cat(d, 'Buy now pay later')]),
}
for key, (_, ds) in segments.items():
    keep = {d['domain'] for d in ds}
    with (OUT / f'segment-{key}.csv').open('w', newline='') as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0]))
        w.writeheader(); w.writerows(r for r in rows if r['domain'] in keep)

def tools(c, pool=seen):
    return Counter(t for d in pool for t in cat(d, c)).most_common(8)

res = {
    'tranco': {'listId': cand['listId'], 'listDate': cand['listDate'], 'scanned': cand['scanned'], 'onShopify': len(cand['matches']), 'sampleRanks': [cand['sample'][0]['rank'], cand['sample'][-1]['rank']]},
    'run': {k: run[k] for k in ('runId', 'status', 'seconds', 'usageTotalUsd', 'chargedEventCounts', 'itemCount', 'startedAt')},
    'requested': summary['requested'], 'pushed': summary['pushed'], 'failed': [f['domain'] for f in summary['failed']],
    'homepageAnalysed': N, 'homepageBotWall': sum(1 for d in items if 'bot-protection' in (d.get('errors') or {}).get('tech', '')),
    'shopifyOnHomepage': sum(1 for d in seen if 'Shopify' in names(d)),
    'tld': Counter(d['domain'].rsplit('.', 1)[-1] for d in items).most_common(10),
    'categories': {c: {'stores': sum(1 for d in seen if cat(d, c)), 'top': tools(c)} for c in ['Marketing automation', 'Reviews', 'Live chat', 'Buy now pay later', 'Cookie compliance', 'A/B Testing', 'Personalisation', 'Analytics', 'Advertising', 'Customer data platform', 'Returns', 'Translation']},
    'emailMarketing': {'any': sum(1 for d in seen if names(d) & EMAIL_TOOLS), 'top': Counter(t for d in seen for t in names(d) & EMAIL_TOOLS).most_common(10)},
    'support': {'any': sum(1 for d in seen if names(d) & SUPPORT), 'top': Counter(t for d in seen for t in names(d) & SUPPORT).most_common(10)},
    'emailHost': Counter(d.get('emailProvider') or 'none found' for d in items).most_common(8),
    'dmarc': dict(Counter(d.get('dmarcPolicy') or 'missing' for d in items)),
    'klaviyo': len(klaviyo),
    'klaviyoDmarc': dict(Counter(d.get('dmarcPolicy') or 'missing' for d in klaviyo)),
    'segments': {k: {'label': l, 'stores': len(ds), 'first': [d['domain'] for d in sorted(ds, key=lambda d: rank.get(d['domain'], 10**9))[:12]]} for k, (l, ds) in segments.items()},
    'priceStarterUsd': round(summary['pushed'] * 0.003 + 0.00005, 4),
}
(OUT / 'results.json').write_text(json.dumps(res, indent=1))
print(json.dumps(res, indent=1))