#!/usr/bin/env python3
"""Step 4 of signal recipe 2 (tech-stack signal): a free second pass over one segment, on a product page.

domain-intel reads one page per domain, the homepage. Review widgets live on product pages, so "no review app on the
homepage" is only a shortlist. This script opens one product page per store in a segment CSV and looks for the
review and email tools there. Shopify-specific: it takes the first product from the store's public /products.json.

Polite by design: one robots.txt, one /products.json?limit=1 and one product page per store, robots.txt respected
(wildcards included), 8 stores at a time. Logged-out public pages only.
Usage:  python3 distribution/analysis/recipe-2-product-check.py klaviyo-no-reviews
Output: out/recipe-2/product-check-<segment>.json  [{ domain, status, product, reviews: [...], email: [...] }]
"""
import concurrent.futures as cf, csv, json, re, sys, urllib.parse, urllib.request
from collections import Counter
from pathlib import Path

OUT = Path(__file__).resolve().parent / 'out' / 'recipe-2'
SEG = sys.argv[1] if len(sys.argv) > 1 else 'klaviyo-no-reviews'
UA = 'Mozilla/5.0 (compatible; locaihost-domain-intel; +https://locaihost.org/)'
UA_TOKEN = 'locaihost-domain-intel'

# Product-specific signals only: the vendor's script host, its Shopify app-embed folder, or its widget markup.
EXT = r'cdn\.shopify\.com/extensions/[0-9a-f-]+/'
REVIEWS = {
    'Judge.me': rf'cdn\.judge\.me|class=["\'][^"\']*\bjdgm-|{EXT}judgeme',
    'Yotpo': rf'staticw2\.yotpo\.com|cdn-widgetsrepository\.yotpo\.com|yotpo-widget-instance|{EXT}yotpo',
    'Okendo': rf'cdn-static\.okendo\.io|data-oke-widget|{EXT}okendo',
    'Loox': rf'loox\.io/widget|{EXT}loox|looxreviews|__loox_reviews',
    'Stamped': rf'cdn1\.stamped\.io|stamped-main-widget|{EXT}stamped',
    'REVIEWS.io': rf'widget\.reviews\.io|widget\.reviews\.co\.uk|{EXT}reviews-co-uk|__reviews_io',
    'Klaviyo Reviews': r'klaviyo-star-rating-widget|id=["\']klaviyo-reviews|__klaviyo_reviews',
    'Trusted Shops': r'widgets\.trustedshops\.com|__trusted_shops',
    'Junip': r'scripts\.juniphq\.com|junip-product-review',
    'Trustpilot': r'widget\.trustpilot\.com|class=["\']trustpilot-widget|trustpilot-product-reviews',
    # Any other Shopify app block with "review" in its name, and the theme's own reviews section.
    'Other review app': r'shopify-block-[a-z0-9]+__[a-z0-9_]*review',
    'Reviews section (own or unidentified)': r'(?:class|id)=["\'][^"\']*(?:product-reviews?\b|pdp-reviews|reviews?-widget|__reviews\b|review-count|review-stars?\b)',
    'Bazaarvoice': r'apps\.bazaarvoice\.com|data-bv-show',
    'PowerReviews': r'ui\.powerreviews\.com',
    'Shopify Product Reviews': r'id=["\']shopify-product-reviews',
    'Ali Reviews': rf'{EXT}ali-reviews|alireviews\.io',
    'Growave': rf'{EXT}growave|growave\.io',
    'Fera': r'cdn\.fera\.ai',
    'Feefo': r'api\.feefo\.com',
}
EMAIL = {
    'Klaviyo': r'static\.klaviyo\.com|a\.klaviyo\.com',
    'Mailchimp': r'chimpstatic\.com|list-manage\.com',
    'Omnisend': r'omnisnippet|omnisrc\.com|omnisend\.com',
    'Privy': r'widget\.privy\.com',
    'Attentive': r'cdn\.attn\.tv',
    'Postscript': r'sdk\.postscript\.io',
    'Brevo': r'sibautomation\.com|brevo\.com',
    'Drip': r'tag\.getdrip\.com',
}


def get(url, limit=3_000_000):
    req = urllib.request.Request(url, headers={'User-Agent': UA, 'Accept': '*/*'})
    with urllib.request.urlopen(req, timeout=20) as r:
        return r.read(limit).decode('utf8', 'replace')


def disallowed(robots, path):
    """True if the robots.txt group for our token (or *) disallows path. Longest match wins; * and $ supported."""
    groups, cur, in_rules = [], None, False
    for line in robots.splitlines():
        line = line.split('#', 1)[0].strip()
        if ':' not in line:
            continue
        k, v = (s.strip() for s in line.split(':', 1))
        k = k.lower()
        if k == 'user-agent':
            if cur is None or in_rules:
                cur, in_rules = {'agents': [], 'rules': []}, False
                groups.append(cur)
            cur['agents'].append(v.lower())
        elif k in ('allow', 'disallow') and cur is not None:
            in_rules = True
            if v:
                cur['rules'].append((k == 'allow', v))
    mine = [g for g in groups if any(a != '*' and a in UA_TOKEN for a in g['agents'])] or [g for g in groups if '*' in g['agents']]
    best = (-1, True)
    for g in mine:
        for allow, pat in g['rules']:
            rx = '^' + re.escape(pat).replace(r'\*', '.*').replace(r'\$', '$')
            if re.match(rx, path) and (len(pat), allow) > best:
                best = (len(pat), allow)
    return not best[1]


def check(domain):
    base = f'https://{domain}'
    try:
        try:
            robots = get(f'{base}/robots.txt', 500_000)
        except Exception:
            robots = ''
        if disallowed(robots, '/products.json'):
            return {'domain': domain, 'status': 'robots.txt disallows /products.json'}
        prods = json.loads(get(f'{base}/products.json?limit=1')).get('products') or []
        if not prods:
            return {'domain': domain, 'status': 'no public products'}
        path = '/products/' + urllib.parse.quote(prods[0]['handle'])
        if disallowed(robots, path):
            return {'domain': domain, 'status': 'robots.txt disallows the product page'}
        page = get(base + path)
        return {'domain': domain, 'status': 'ok', 'product': path,
                'reviews': sorted(k for k, p in REVIEWS.items() if re.search(p, page, re.I)),
                'email': sorted(k for k, p in EMAIL.items() if re.search(p, page, re.I))}
    except Exception as e:
        return {'domain': domain, 'status': f'not read ({type(e).__name__})'}


rows = list(csv.DictReader((OUT / f'segment-{SEG}.csv').open()))
domains = [r['domain'] for r in rows]
saved = OUT / f'product-check-{SEG}.json'
if '--saved' in sys.argv:  # rebuild the confirmed CSV from the last check without visiting the stores again
    res = json.loads(saved.read_text())
else:
    with cf.ThreadPoolExecutor(8) as ex:
        res = list(ex.map(check, domains))
    saved.write_text(json.dumps(res, indent=1))
ok = [r for r in res if r['status'] == 'ok']
# The confirmed segment: stores whose product page also lacks the missing tool. Unread stores are left out.
missing = 'reviews' if SEG.endswith('no-reviews') else 'email'
confirmed = {r['domain'] for r in ok if not r[missing]}
with (OUT / f'segment-{SEG}-confirmed.csv').open('w', newline='') as f:
    w = csv.DictWriter(f, fieldnames=list(rows[0]))
    w.writeheader(); w.writerows(r for r in rows if r['domain'] in confirmed)
print(f'{SEG}: {len(domains)} stores, product page read on {len(ok)}; '
      f'review app there on {sum(1 for r in ok if r["reviews"])}, email tool there on {sum(1 for r in ok if r["email"])}; '
      f'confirmed (no {missing} tool on either page): {len(confirmed)}')
print('reviews:', Counter(t for r in ok for t in r['reviews']).most_common())
print('email:', Counter(t for r in ok for t in r['email']).most_common())
print('status:', Counter(r['status'] for r in res).most_common())