#!/usr/bin/env python3
"""UK late payers 2026: analysis of every Procurement Act payments compliance notice (UK17) on Find a Tender.

Input  out/late-payers-2026/raw.json      the uk-procurement-act Actor's dataset (payments-compliance, full history)
       out/late-payers-2026/regions.json  each notice's authority address region code (late-payers-2026-fetch.py)
Output out/late-payers-2026/results.json  every figure the post uses
       out/late-payers-2026/tables.md     the same as readable tables

Rules: organisations only (the Actor never reads the approving officer's name or the narrative); figures are what each
authority reported, not our judgement. Run: python3 distribution/analysis/late-payers-2026.py
"""
import json
import re
import statistics as st
from collections import Counter, defaultdict
from pathlib import Path

HERE = Path(__file__).resolve().parent
OUT = HERE / 'out' / 'late-payers-2026'
rows = json.loads((OUT / 'raw.json').read_text())
regions = json.loads((OUT / 'regions.json').read_text())

# ---------------------------------------------------------------- sector, from the authority's name
HOUSING = re.compile(r"housing|homes\b|\bhomes|clarion|peabody|guinness|sanctuary|riverside group|notting hill genesis|"
                     r"orbit group|stonewater|home group|anchor hanover|gentoo|torus|karbon|midland heart|waythrough|livv|"
                     r"bernicia|aster group|acis |amplius|saxon weald|brunelcare|extracare|vivid|magna|flagship|livewest|"
                     r"greensquareaccord|thirteen|alpha rsl|poplar|rooftop|eastlight|citizen|connexus|progress housing|"
                     r"great places|believe|calico|coastline|hobstones|vico|greatwell|cross keys|havebury|two rivers|"
                     r"south lakes|saffron|halton housing|hafod|cadwyn|bron afon|habinteg|newlon|muir group|orwell|"
                     r"cottsway|broadacres|ring stones|mhs homes|accent housing|salix|ongo|wandle|worthing homes", re.I)
NOT_HOUSING = re.compile(r"homes england|ministry of housing|lancashire renewables", re.I)
POLICE_FIRE = re.compile(r"police|fire|chief constable|policing", re.I)
UNI = re.compile(r"universit|college|prifysgol|birkbeck|school of tropical|higher education corporation", re.I)
NHS = re.compile(r"\bnhs\b|ambulance service|integrated care board|healthcare quality improvement|"
                 r"supply chain coordination|genomics england", re.I)
NATIONAL = re.compile(r"forestry commission|government property agency|homes england|information commissioner|"
                      r"national crime agency|ofcom|ofgem|financial conduct authority|royal household|general medical council|"
                      r"general dental council|general pharmaceutical council|construction industry training board|"
                      r"financial services compensation scheme|national oceanography centre|npl management|"
                      r"national nuclear laboratory|catapult|pirbright|waste and resources action|uk sports institute|"
                      r"pool re|reclaim fund|jisc|global commercial services", re.I)
NOT_COUNCIL = re.compile(r"general (medical|dental|pharmaceutical) council|nursing and midwifery council|"
                         r"london councils|association of north east councils|financial reporting council", re.I)
COUNCIL = re.compile(r"council\b|london borough of|royal borough of|^borough of|mayor and commonalty", re.I)
TRANSPORT = re.compile(r"trains limited|railway|rail infrastructure|transport for|merseytravel|national highways|"
                       r"high speed two|ny highways|dft operator|rail ltd", re.I)
OTHER_LOCAL = re.compile(r"combined (county )?authority|greater london authority|national park authority|"
                         r"development corporation|pensions authority|regional park|lgps central|border to coast|"
                         r"purchasing organisation|espo|london councils|association of north east councils|"
                         r"southern universities management", re.I)


def sector(r):
    n = r['authorityName']
    if HOUSING.search(n) and not NOT_HOUSING.search(n):
        return 'Housing associations and council housing companies'
    if NHS.search(n):
        return 'NHS bodies'
    if POLICE_FIRE.search(n):
        return 'Police and fire'
    if OTHER_LOCAL.search(n):
        return 'Combined authorities and other local bodies'
    if UNI.search(n):
        return 'Universities and colleges'
    if COUNCIL.search(n) and not NOT_COUNCIL.search(n):
        return 'Councils'
    if TRANSPORT.search(n):
        return 'Transport operators'
    if NATIONAL.search(n):
        return 'Central government and national bodies'
    if r['authorityType'] == 'Public authority - central government':
        return 'Central government and national bodies'
    return 'Council-owned companies and other bodies'


# ---------------------------------------------------------------- region, from the authority's own ITL/NUTS code
ITL1 = {'UKC': 'North East', 'UKD': 'North West', 'UKE': 'Yorkshire and the Humber', 'UKF': 'East Midlands',
        'UKG': 'West Midlands', 'UKH': 'East of England', 'UKI': 'London', 'UKJ': 'South East', 'UKK': 'South West',
        'UKL': 'Wales', 'UKM': 'Scotland', 'UKN': 'Northern Ireland'}


def region(r):
    code = (regions.get(r['noticeId'], {}).get('region') or '').upper()
    return ITL1.get(code[:3])


# ---------------------------------------------------------------- report sets
for r in rows:
    r['sector'] = sector(r)
    r['region'] = region(r)
    r['months'] = round((int(r['paymentPeriodEnd'][:4]) * 12 + int(r['paymentPeriodEnd'][5:7]))
                        - (int(r['paymentPeriodStart'][:4]) * 12 + int(r['paymentPeriodStart'][5:7])) + 1)

ranked = [r for r in rows if not r['emptyReport'] and r['paidWithin30DaysPct'] is not None]
short_form = [r for r in rows if not r['emptyReport'] and r['paidWithin30DaysPct'] is None]
empty = [r for r in rows if r['emptyReport']]
authorities = {r['authorityKey'] for r in rows}

# The main period: October 2025 to March 2026 (the first full six-month period most authorities report on).
MAIN = ('2025-10-01', '2026-03-31')
main = [r for r in ranked if (r['paymentPeriodStart'], r['paymentPeriodEnd']) == MAIN]
main_all = [r for r in main if r['contractsCovered'].startswith('All contracts')]


def pct(v, n):
    return round(100 * v / n, 1) if n else None


def med(xs):
    return round(st.median(xs), 1) if xs else None


def summary(rs):
    p = [r['paidWithin30DaysPct'] for r in rs]
    d = [r['averageDaysToPay'] for r in rs if r['averageDaysToPay'] is not None]
    return {
        'reports': len(rs),
        'medianPaidWithin30': med(p),
        'meanPaidWithin30': round(st.mean(p), 1) if p else None,
        'medianAvgDays': med(d),
        'atLeast95': sum(x >= 95 for x in p), 'atLeast95Pct': pct(sum(x >= 95 for x in p), len(p)),
        'under80': sum(x < 80 for x in p), 'under80Pct': pct(sum(x < 80 for x in p), len(p)),
        'under50': sum(x < 50 for x in p), 'under50Pct': pct(sum(x < 50 for x in p), len(p)),
        'all100': sum(x == 100 for x in p),
        'medianPaid61Plus': med([r['paid61PlusDaysPct'] for r in rs if r['paid61PlusDaysPct'] is not None]),
        'medianNotPaid': med([r['duePaymentsNotMadePct'] for r in rs if r['duePaymentsNotMadePct'] is not None]),
    }


def row_out(r):
    return {k: r[k] for k in ['authorityName', 'sector', 'region', 'paidWithin30DaysPct', 'paid31To60DaysPct',
                              'paid61PlusDaysPct', 'averageDaysToPay', 'duePaymentsNotMadePct', 'paymentPeriodStart',
                              'paymentPeriodEnd', 'contractsCovered', 'noticeId', 'noticeUrl']}


# Worst / best: main period, reports covering all contracts (comparable scope).
worst = sorted(main_all, key=lambda r: (r['paidWithin30DaysPct'], -(r['averageDaysToPay'] or 0)))[:15]
best_pool = [r for r in main_all if r['paidWithin30DaysPct'] >= 99]
best = sorted(main_all, key=lambda r: (-r['paidWithin30DaysPct'], r['averageDaysToPay'] if r['averageDaysToPay'] else 999))[:15]
slowest_avg = sorted([r for r in main_all if r['averageDaysToPay'] is not None], key=lambda r: -r['averageDaysToPay'])[:10]

# By sector and region (main period, all contract scopes; one report per authority in that period).
by_sector = defaultdict(list)
for r in main:
    by_sector[r['sector']].append(r)
sector_table = sorted(({'sector': s, **summary(rs)} for s, rs in by_sector.items()), key=lambda x: -x['medianPaidWithin30'])

councils = [r for r in main if r['sector'] == 'Councils']
by_region = defaultdict(list)
for r in councils:
    if r['region']:
        by_region[r['region']].append(r)
region_table = sorted(({'region': g, **summary(rs)} for g, rs in by_region.items()), key=lambda x: -x['medianPaidWithin30'])

# Change between periods (the Actor's own comparison: the authority's previous reporting period).
compared = [r for r in rows if r['paymentTrend'] in ('worse', 'better', 'unchanged')]
trend = Counter(r['paymentTrend'] for r in compared)
movers = sorted([r for r in compared if r['paidWithin30DaysChange'] is not None], key=lambda r: r['paidWithin30DaysChange'])
change_rows = [{**row_out(r), 'previousPaidWithin30DaysPct': r['previousPaidWithin30DaysPct'],
                'paidWithin30DaysChange': r['paidWithin30DaysChange'], 'averageDaysToPayChange': r['averageDaysToPayChange'],
                'previousPeriodStart': r['previousPeriodStart'], 'previousPeriodEnd': r['previousPeriodEnd'],
                'previousAverageDaysToPay': r['previousAverageDaysToPay'], 'paymentTrend': r['paymentTrend']} for r in movers]

# Late share: the notices give percentages of invoices, never invoice counts or values.
late = summary(main)
late_all = summary(main_all)
dist = Counter()
for r in main:
    p = r['paidWithin30DaysPct']
    band = '100' if p == 100 else '95-99' if p >= 95 else '90-94' if p >= 90 else '80-89' if p >= 80 else '50-79' if p >= 50 else '0-49'
    dist[band] += 1

by_scope = defaultdict(list)
for r in main:
    by_scope[r['contractsCovered']].append(r)
scope_table = [{'scope': k, **summary(v)} for k, v in by_scope.items()]
latest_rs = [r for r in ranked if (r['paymentPeriodStart'], r['paymentPeriodEnd']) == ('2026-04-01', '2026-09-30')]
latest_low = sorted(latest_rs, key=lambda r: r['paidWithin30DaysPct'])[:8]
narrow_zero = [row_out(r) for r in main if not r['contractsCovered'].startswith('All') and r['paidWithin30DaysPct'] == 0]

res = {
    'snapshot': '2026-10-14',
    'totals': {
        'currentReports': len(rows), 'authorities': len(authorities),
        'ranked': len(ranked), 'shortForm': len(short_form), 'empty': len(empty),
        'byPeriod': Counter(f"{r['paymentPeriodStart']}..{r['paymentPeriodEnd']}" for r in rows),
        'byScope': Counter(r['contractsCovered'] for r in rows),
        'byAuthorityType': Counter(r['authorityType'] for r in rows),
        'bySector': Counter(r['sector'] for r in rows),
        'mainPeriodRanked': len(main), 'mainPeriodAllContracts': len(main_all),
        'regionKnown': sum(1 for r in rows if r['region']),
        'byRegionAll': Counter(r['region'] for r in rows),
    },
    'late': late, 'lateAllContracts': late_all, 'distribution': dist,
    'worst': [row_out(r) for r in worst],
    'best': [row_out(r) for r in best], 'bestPoolAtLeast99': len(best_pool),
    'slowestAverage': [row_out(r) for r in slowest_avg],
    'sectors': sector_table,
    'councilRegions': region_table,
    'trend': {'compared': len(compared), **trend},
    'changes': change_rows,
    'latestPeriod': summary([r for r in ranked if (r['paymentPeriodStart'], r['paymentPeriodEnd']) == ('2026-04-01', '2026-09-30')]),
    'scopes': scope_table,
    'latestLowest': [row_out(r) for r in latest_low],
    'narrowScopeZero': narrow_zero,
    'unclassifiedOther': sorted({r['authorityName'] for r in rows if r['sector'] == 'Council-owned companies and other bodies'}),
}
(OUT / 'results.json').write_text(json.dumps(res, indent=1, ensure_ascii=False))


# ---------------------------------------------------------------- tables.md
def fmt(x):
    return '' if x is None else (f'{x:g}' if isinstance(x, (int, float)) else str(x))


md = ['# UK late payers 2026: tables', '', f"Snapshot {res['snapshot']}. {len(rows)} current reports from {len(authorities)} authorities.", '']
md += ['## Worst (Oct 2025 to Mar 2026, all contracts)', '', '| # | Authority | Sector | Within 30 days % | 61+ days % | Avg days |', '|---:|---|---|---:|---:|---:|']
md += [f"| {i} | {r['authorityName']} | {r['sector']} | {fmt(r['paidWithin30DaysPct'])} | {fmt(r['paid61PlusDaysPct'])} | {fmt(r['averageDaysToPay'])} |" for i, r in enumerate(worst, 1)]
md += ['', '## Best', '', '| # | Authority | Sector | Within 30 days % | Avg days |', '|---:|---|---|---:|---:|']
md += [f"| {i} | {r['authorityName']} | {r['sector']} | {fmt(r['paidWithin30DaysPct'])} | {fmt(r['averageDaysToPay'])} |" for i, r in enumerate(best, 1)]
md += ['', '## Sectors (Oct 2025 to Mar 2026)', '', '| Sector | Reports | Median % within 30 | Median avg days | ≥95% | <80% |', '|---|---:|---:|---:|---:|---:|']
md += [f"| {s['sector']} | {s['reports']} | {fmt(s['medianPaidWithin30'])} | {fmt(s['medianAvgDays'])} | {s['atLeast95Pct']}% | {s['under80Pct']}% |" for s in sector_table]
md += ['', '## Councils by region', '', '| Region | Councils | Median % within 30 | Median avg days | <80% |', '|---|---:|---:|---:|---:|']
md += [f"| {s['region']} | {s['reports']} | {fmt(s['medianPaidWithin30'])} | {fmt(s['medianAvgDays'])} | {s['under80']} |" for s in region_table]
md += ['', '## Change since previous report', '', f"{res['trend']}", '', '| Authority | Previous | Now | Change (pts) | Avg days change | Trend |', '|---|---:|---:|---:|---:|---|']
md += [f"| {r['authorityName']} | {fmt(r['previousPaidWithin30DaysPct'])} | {fmt(r['paidWithin30DaysPct'])} | {fmt(r['paidWithin30DaysChange'])} | {fmt(r['averageDaysToPayChange'])} | {r['paymentTrend']} |" for r in change_rows]
(OUT / 'tables.md').write_text('\n'.join(md) + '\n')

if __name__ == '__main__':
    t = res['totals']
    print(json.dumps({k: t[k] for k in ['currentReports', 'authorities', 'ranked', 'shortForm', 'empty', 'mainPeriodRanked', 'mainPeriodAllContracts', 'regionKnown']}))
    print('sectors', dict(t['bySector']))
    print('late', late)
    print('lateAll', late_all)
    print('dist', dict(dist))
    print('trend', res['trend'])
    print('latest', res['latestPeriod'])
    print('other:', res['unclassifiedOther'])