#!/usr/bin/env python3
"""Fetch the inputs for the UK late-payers data story (2026).

1. The Actor's dataset (uk-procurement-act, payments-compliance only, full history) is fetched separately with:
     POST https://api.apify.com/v2/acts/m2XOuQfgiHYT80ZWR/run-sync-get-dataset-items
     {"noticeTypes":["payments-compliance"],"daysBack":365,"maxResults":0}
   and saved as out/late-payers-2026/raw.json (675 current reports on 2026-10-14).

2. This script reads Find a Tender's open organisationReleasePackages API (the same source the Actor uses) to get
   two things the Actor deliberately leaves out of its rows:
     - the contracting authority's *organisation* address region / postcode area (for a regional view), and
     - the list of statistic measure names (to check whether any report gives invoice values).
   It keeps only those fields. It never stores `notes`, `approvedBy`, `contactPoint` or the street address,
   and never prints statistic values.

Also stores, per notice, only the published statistics, period, dimension, buyer name and the id of the notice a
correction replaces (out/late-payers-2026/fts-statistics.json), so late-payers-2026-check.py can re-derive every
figure from Find a Tender's own rows without going through the Actor.

Output: out/late-payers-2026/regions.json  {noticeId: {region, countryName, postcodeArea}}
        out/late-payers-2026/measures.json {measureName: count}
"""
import json
import re
import time
import urllib.request
from collections import Counter
from pathlib import Path

OUT = Path(__file__).parent / 'out' / 'late-payers-2026'
URL = 'https://www.find-tender.service.gov.uk/api/1.0/organisationReleasePackages?updatedFrom=2025-02-24T00:00:00&limit=100'


def get(url):
    req = urllib.request.Request(url, headers={'User-Agent': 'locaihost-data-story/1.0 (+https://locaihost.org)'})
    for attempt in range(5):
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                return json.load(r)
        except Exception:  # noqa: BLE001
            time.sleep(10 * (attempt + 1))
    raise RuntimeError('fetch failed: ' + url)


def main():
    OUT.mkdir(parents=True, exist_ok=True)
    regions, measures, stats_out = {}, Counter(), {}
    url, pages = URL, 0
    while url:
        pkg = get(url)
        pages += 1
        rels = pkg.get('releases') or []
        for r in rels:
            buyer_id = (r.get('buyer') or {}).get('id')
            party = next((p for p in r.get('parties') or [] if p.get('id') == buyer_id), None)
            if party is None:
                party = next((p for p in r.get('parties') or [] if 'buyer' in (p.get('roles') or [])), {})
            addr = party.get('address') or {}
            pc = (addr.get('postalCode') or '').strip().upper()
            m = re.match(r'^([A-Z]{1,2})\d', pc)
            regions[r['id']] = {
                'region': addr.get('region'),
                'countryName': addr.get('countryName'),
                'postcodeArea': m.group(1) if m else None,
            }
            comp = []
            for c in r.get('transactionCompliance') or []:
                stats = {}
                for st in c.get('statistics') or []:
                    measures[st.get('measure')] += 1
                    stats[st.get('measure')] = st.get('value')
                comp.append({'period': c.get('period'), 'dimension': c.get('dimension'), 'statistics': stats,
                             'documentIds': [d.get('id') for d in c.get('documents') or []]})
            stats_out[r['id']] = {'date': r.get('date'), 'previousNotice': r.get('previousNotice'),
                                  'buyerName': (r.get('buyer') or {}).get('name'), 'compliance': comp}
        url = (pkg.get('links') or {}).get('next') if rels else None
        time.sleep(2)
    (OUT / 'regions.json').write_text(json.dumps(regions, indent=1))
    (OUT / 'measures.json').write_text(json.dumps(measures, indent=1))
    (OUT / 'fts-statistics.json').write_text(json.dumps(stats_out, indent=1))
    print(f'pages={pages} releases={len(regions)} measures={len(measures)}')


if __name__ == '__main__':
    main()