#!/usr/bin/env python3
"""Signal recipe 4 (content signal): numbers for the article, from out/recipe-4/videos.json (recipe-4-run.mjs).
Counts which tools, companies and topics the newest 50 videos of 6 B2B sales channels talk about. Plain text matching
on the captions, no model: tool and company names are matched case-sensitively (captions capitalise names), with the
caption spellings we found for a few of them (Salesoft, Raplit, chat GPT); topics are phrase lists, matched ignoring case.
No channel's own brand is in the list (Close, the CRM, is left out: "close" is too common a word).
Writes out/recipe-4/results.json (every figure the article quotes), tool-mentions.csv and topic-mentions.csv (counts per
channel), videos.csv (one row per video: public metadata and counts, no transcript text) and chunks.jsonl (the RAG
export, local only: it holds transcript text, which stays the creators' work).
Run: python3 distribution/analysis/recipe-4-analysis.py
"""
import csv, json, re, statistics
from collections import Counter, defaultdict
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = HERE / 'out' / 'recipe-4'
d = json.loads((OUT / 'videos.json').read_text())
runs = json.loads((OUT / 'runs.json').read_text())
videos = [v for v in d['items'] if v.get('hasTranscript')]
free = [v for v in d['items'] if not v.get('hasTranscript')]
CH = {c['handle'].lower(): c['name'] for c in d['channels']}
chan = lambda v: CH[v['sourceInput'].lower()]

# name -> (group, regex). Case-sensitive. Ambiguous words get a guard (Ramp up, mid-sentence Notion) or are left out
# (Instantly, Close, Harvey: ordinary words or first names too often).
TOOLS = {
    # Sales and go-to-market software
    'Salesforce': ('Sales software', r'\bSalesforce\b'),
    'LinkedIn': ('Sales software', r'\bLinked[Ii]n\b'),
    'HubSpot': ('Sales software', r'\bHub[Ss]pot\b'),
    'Gong': ('Sales software', r'\bGong\b'),
    'Clay': ('Sales software', r'\bClay\b'),
    'Outreach': ('Sales software', r'(?<=[a-z,] )Outreach\b'),
    'Salesloft': ('Sales software', r'\bSales ?(?:loft|oft)\b'),
    'Apollo': ('Sales software', r'\bApollo\b'),
    'ZoomInfo': ('Sales software', r'\bZoom ?[Ii]nfo\b'),
    'Pipedrive': ('Sales software', r'\bPipedrive\b'),
    'Intercom': ('Sales software', r'\bIntercom\b'),
    'Zendesk': ('Sales software', r'\bZendesk\b'),
    # AI models and app builders
    'Claude / Anthropic': ('AI tools', r'\bClaude\b|\bAnthropic\b'),
    'ChatGPT / OpenAI': ('AI tools', r'\bChat ?GPT\b|\bchat GPT\b|\bOpen ?AI\b|\bGPT[- ]?\d'),
    'Gemini': ('AI tools', r'\bGemini\b'),
    'Copilot': ('AI tools', r'\bCopilot\b'),
    'Perplexity': ('AI tools', r'(?<=[a-z,] )Perplexity\b'),
    'Replit': ('AI tools', r'\bRepl[ie]t\b|\bRaplit\b'),
    'Lovable': ('AI tools', r'(?<=[a-z,] )Lovable\b'),
    'Cursor': ('AI tools', r'\bCursor\b'),
    # Other software companies
    'Google': ('Other software', r'\bGoogle\b'),
    'Slack': ('Other software', r'\bSlack\b'),
    'Zoom': ('Other software', r'\bZoom\b(?! ?[Ii]nfo)'),
    'Shopify': ('Other software', r'\bShopify\b'),
    'Microsoft': ('Other software', r'\bMicrosoft\b'),
    'Snowflake': ('Other software', r'\bSnowflake\b'),
    'Stripe': ('Other software', r'\bStripe\b'),
    'Ramp': ('Other software', r'\bRamp\b(?! up)'),
    'Notion': ('Other software', r'(?<=[a-z,] )Notion\b'),
    'Zapier': ('Other software', r'\bZapier\b'),
}
TOPICS = {
    'AI agents': ['ai agent', 'ai agents', 'agentic', 'ai sdr', 'ai sdrs', 'agents'],
    'Cold calling': ['cold call', 'cold calls', 'cold calling', 'phone prospecting', 'dial', 'dials', 'dialing'],
    'Cold email': ['cold email', 'cold emails', 'email sequence', 'sequences', 'subject line', 'reply rate'],
    'LinkedIn prospecting': ['linkedin', 'sales navigator', 'connection request', 'inmail'],
    'Discovery': ['discovery call', 'discovery calls', 'discovery questions', 'discovery'],
    'Objection handling': ['objection', 'objections', 'pushback', 'brush-off', 'brush off'],
    'Pricing': ['pricing', 'price increase', 'discount', 'discounting', 'usage-based', 'per seat', 'seat-based'],
    'Negotiation': ['negotiation', 'negotiate', 'negotiating', 'procurement'],
    'Pipeline and forecasting': ['pipeline', 'forecast', 'forecasting', 'deal review'],
    'Quota and compensation': ['quota', 'quotas', 'commission', 'comp plan', 'ote', 'on-target earnings', 'accelerator'],
    'Hiring and ramping reps': ['hiring', 'hire', 'onboarding new reps', 'ramp time', 'recruit', 'recruiting'],
    'Churn, retention and expansion': ['churn', 'retention', 'renewal', 'renewals', 'expansion', 'upsell', 'net revenue retention', 'nrr'],
    'Product-led growth': ['product-led', 'product led', 'plg', 'free trial', 'freemium', 'self-serve', 'self serve'],
    'Ideal customer profile': ['ideal customer profile', 'icp'],
    'Founder-led sales': ['founder-led', 'founder led'],
    'Fundraising and valuations': ['fundraising', 'raise money', 'series a', 'series b', 'valuation', 'venture capital', 'vcs'],
}
tool_re = {k: re.compile(r, re.M) for k, (_, r) in TOOLS.items()}
topic_re = {k: re.compile(r'\b(?:' + '|'.join(re.escape(p) for p in ps) + r')\b', re.I) for k, ps in TOPICS.items()}
# A topic counts for a video when it comes up at least this often; one passing mention isn't the subject of a video.
TOPIC_MIN = 3

def own(name, channel):
    return False  # none of the names above is one of the six channels' own brand

# Intros, outros and ad reads repeat word for word across a channel's episodes, and an ad for one tool often names
# another ("we ripped out Gong"). Any run of WINDOW words found in REPEAT_MIN or more videos of the same channel is
# set aside as a repeated passage; tools and topics are counted on the rest (the conversation), and tool mentions
# inside repeated passages are counted separately.
WINDOW, REPEAT_MIN = 12, 3
def words(t):
    return [(m.group(0), re.sub(r'[^a-z0-9]', '', m.group(0).lower())) for m in re.finditer(r'\S+', t)]
by_ch = defaultdict(list)
for v in videos:
    v['_w'] = words(v['transcript'])
    by_ch[chan(v)].append(v)
for c, vs in by_ch.items():
    seen = defaultdict(set)
    for v in vs:
        keys = [w for _, w in v['_w']]
        for k in range(len(keys) - WINDOW + 1):
            seen[' '.join(keys[k:k + WINDOW])].add(v['videoId'])
    for v in vs:
        keys = [w for _, w in v['_w']]
        cover = [False] * len(keys)
        for k in range(len(keys) - WINDOW + 1):
            if len(seen[' '.join(keys[k:k + WINDOW])]) >= REPEAT_MIN:
                for j in range(k, k + WINDOW):
                    cover[j] = True
        v['_conv'] = ' '.join(o for (o, _), cv in zip(v['_w'], cover) if not cv)
        v['_rep'] = ' '.join(o for (o, _), cv in zip(v['_w'], cover) if cv)
        v['_repWords'] = sum(cover)

rows, tool_ch, topic_ch = [], defaultdict(Counter), defaultdict(Counter)
rep_vids, rep_vids_ch = Counter(), defaultdict(Counter)
tool_vids, tool_mentions, topic_vids = Counter(), Counter(), Counter()
tool_vids_ch, topic_vids_ch = defaultdict(Counter), defaultdict(Counter)
for v in videos:
    c, t = chan(v), v['_conv']
    for k, r in tool_re.items():
        if r.search(v['_rep']):
            rep_vids[k] += 1; rep_vids_ch[k][c] += 1
    tc = {k: len(r.findall(t)) for k, r in tool_re.items() if not own(k, c)}
    tc = {k: n for k, n in tc.items() if n}
    pc = {k: len(r.findall(t)) for k, r in topic_re.items()}
    pc = {k: n for k, n in pc.items() if n >= TOPIC_MIN}
    v['_tools'], v['_topics'] = tc, pc
    for k, n in tc.items():
        tool_vids[k] += 1; tool_mentions[k] += n; tool_ch[k][c] += n; tool_vids_ch[k][c] += 1
    for k, n in pc.items():
        topic_vids[k] += 1; topic_ch[k][c] += 1; topic_vids_ch[k][c] += 1

channels = [c['name'] for c in d['channels']]
per_ch = Counter(chan(v) for v in videos)

with (OUT / 'tool-mentions.csv').open('w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['tool', 'group', 'videos_mentioning', 'mentions'] + [f'videos_{c}' for c in channels] + ['videos_with_it_in_ads_intros_or_outros'])
    for k, n in tool_vids.most_common():
        w.writerow([k, TOOLS[k][0], n, tool_mentions[k]] + [tool_vids_ch[k][c] for c in channels] + [rep_vids[k]])
with (OUT / 'topic-mentions.csv').open('w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['topic', f'videos_with_{TOPIC_MIN}_or_more_mentions'] + [f'videos_{c}' for c in channels])
    for k, n in topic_vids.most_common():
        w.writerow([k, n] + [topic_vids_ch[k][c] for c in channels])
with (OUT / 'videos.csv').open('w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['channel', 'video_id', 'url', 'title', 'published', 'duration_seconds', 'views', 'words', 'auto_captions', 'tools_mentioned', 'topics'])
    for v in sorted(videos, key=lambda v: (chan(v), v['publishedAt']), reverse=False):
        w.writerow([chan(v), v['videoId'], v['url'], v['title'], v['publishedAt'][:10], v['durationSeconds'], v.get('viewCount'), v['wordCount'],
                    v['isAutoGenerated'], '; '.join(f'{k} ({n})' for k, n in sorted(v['_tools'].items(), key=lambda x: -x[1])),
                    '; '.join(sorted(v['_topics'], key=lambda k: -v['_topics'][k]))])

# RAG export: one JSON line per chunk, with a link that opens the video at the chunk's start. Local only.
n_chunks = 0
with (OUT / 'chunks.jsonl').open('w') as f:
    for v in videos:
        for ch in v.get('chunks') or []:
            f.write(json.dumps({
                'id': f"{v['videoId']}-{ch['index']}", 'videoId': v['videoId'], 'channel': chan(v), 'title': v['title'],
                'publishedAt': v['publishedAt'][:10], 'start': ch['start'], 'end': ch['end'],
                'url': f"https://www.youtube.com/watch?v={v['videoId']}&t={int(ch['start'])}s", 'text': ch['text'],
            }, ensure_ascii=False) + '\n')
            n_chunks += 1

# Head-to-head pairs a reader would ask about: videos that mention each, by channel.
pairs = {}
for a, b in [('Claude / Anthropic', 'ChatGPT / OpenAI'), ('Gong', 'Clay'), ('Salesforce', 'HubSpot'), ('Replit', 'Lovable')]:
    pairs[f'{a} vs {b}'] = {c: [tool_vids_ch[a][c], tool_vids_ch[b][c]] for c in channels}

# Quarter trend for the two topics that move: share of each quarter's videos that discuss them.
def quarter(v):
    y, m = int(v['publishedAt'][:4]), int(v['publishedAt'][5:7])
    return f'{y}-Q{(m - 1) // 3 + 1}'
q = defaultdict(list)
for v in videos:
    q[quarter(v)].append(v)
trend = {k: {'videos': len(vs), 'aiAgents': sum('AI agents' in v['_topics'] for v in vs), 'coldCalling': sum('Cold calling' in v['_topics'] for v in vs),
             'claude': sum('Claude / Anthropic' in v['_tools'] for v in vs), 'chatgpt': sum('ChatGPT / OpenAI' in v['_tools'] for v in vs)}
         for k, vs in sorted(q.items())}

first, second = runs[0], runs[1]
res = {
    'snapshot': first['startedAt'][:10],
    'runs': [{k: r[k] for k in ('label', 'runId', 'status', 'seconds', 'usageTotalUsd', 'chargedEventCounts', 'itemCount')} for r in runs],
    'videosAsked': len(d['items']), 'transcripts': len(videos), 'freeRows': len(free),
    'autoCaptions': sum(bool(v['isAutoGenerated']) for v in videos),
    'words': sum(v['wordCount'] for v in videos), 'hours': round(sum(int(v['durationSeconds']) for v in videos) / 3600, 1),
    'chunks': n_chunks,
    'channels': {c: {'videos': per_ch[c],
                     'oldest': min(v['publishedAt'][:10] for v in videos if chan(v) == c),
                     'newest': max(v['publishedAt'][:10] for v in videos if chan(v) == c),
                     'medianMinutes': round(statistics.median(int(v['durationSeconds']) for v in videos if chan(v) == c) / 60, 1),
                     'words': sum(v['wordCount'] for v in videos if chan(v) == c)} for c in channels},
    'repeatedPassages': {'window': WINDOW, 'minVideos': REPEAT_MIN, 'words': sum(v['_repWords'] for v in videos),
                         'byChannel': {c: sum(v['_repWords'] for v in by_ch[c]) for c in channels}},
    'tools': [{'tool': k, 'group': TOOLS[k][0], 'videos': n, 'mentions': tool_mentions[k], 'channels': sum(1 for c in channels if tool_vids_ch[k][c]),
               'videosInRepeatedPassages': rep_vids[k], 'repeatedByChannel': {c: rep_vids_ch[k][c] for c in channels if rep_vids_ch[k][c]}}
              for k, n in tool_vids.most_common()],
    'topics': [{'topic': k, 'videos': n, 'byChannel': {c: topic_vids_ch[k][c] for c in channels}} for k, n in topic_vids.most_common()],
    'pairs': pairs, 'trend': trend, 'topicMin': TOPIC_MIN,
    'priceStarterUsd': round(len(videos) * 0.003 + 0.00005, 4),
}
(OUT / 'results.json').write_text(json.dumps(res, indent=1, ensure_ascii=False))
print(json.dumps({k: res[k] for k in res if k not in ('tools', 'topics')}, indent=1, ensure_ascii=False))
for t in res['tools']:
    print(f"{t['tool']:22} {t['group']:14} vids={t['videos']:4} mentions={t['mentions']:5} channels={t['channels']} rep={t['videosInRepeatedPassages']} {t['repeatedByChannel']}")
for t in res['topics']:
    print(f"{t['topic']:32} vids={t['videos']:4} {t['byChannel']}")