"""Reproduce AMR's complete U.S. crude versus petroleum trade comparison.

Doc: AMR fixed oil-trade analysis
Status: October 2026 MER source vintage, checked November 14, 2026
Owner: American Mineral Registry
Last verified: 2026-11-14
Depends on: pinned EIA T03.03A/B/E CSV files and evidence-contract.json
Run: python3 analysis.py, standard library only, no network or source mutation.
"""
from pathlib import Path
from decimal import Decimal,getcontext
from fractions import Fraction
import csv,json,hashlib,calendar
getcontext().prec=34
ROOT=Path(__file__).resolve().parent;DATA=ROOT/'data';QA=ROOT/'qa';DATA.mkdir(exist_ok=True);QA.mkdir(exist_ok=True)
D=Decimal
URL={s:'https://www.eia.gov/totalenergy/data/browser/csv.php?tbl='+s for s in ['T03.03A','T03.03B','T03.03E']}
MSNS=['COIMPUS','COEXPUS','PAIMPUS','PAEXPUS','PANIPUS']
CHECKS=[]
def ck(name,test):CHECKS.append(name);assert test,name
def write(name,rows):
    ck(name+' complete',bool(rows) and all(set(r)==set(rows[0]) for r in rows) and all(v is not None and str(v)!='' for r in rows for v in r.values()))
    with (DATA/name).open('w',newline='') as f:
        w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows)
def source():
    raw={};selected=[]
    for table,msns in [('T03.03B',['COIMPUS','PAIMPUS']),('T03.03E',['COEXPUS','PAEXPUS']),('T03.03A',['PANIPUS'])]:
        for r in csv.DictReader((ROOT/'sources'/f'{table}.csv').open()):
            if r['MSN'] not in msns:continue
            key=(r['MSN'],r['YYYYMM']);ck('unique source '+str(key),key not in raw);raw[key]=r
    return raw
def panel(raw,periods,frequency):
    result=[]
    for period in periods:
        rows=[raw[(msn,period)] for msn in MSNS]
        for r in rows:ck('source unit '+str((r['MSN'],period)),r['Unit']=='Thousand Barrels per Day')
        ci,ce,pi,pe,published=[D(r['Value']) for r in rows];ck('positive gross '+period,all(x>=0 for x in [ci,ce,pi,pe]))
        noni=pi-ci;none=pe-ce;cn=ci-ce;nonnet=noni-none;pn=pi-pe
        ck('non-crude gross nonnegative '+period,noni>=0 and none>=0)
        ck('accounting identity '+period,cn+nonnet==pn)
        ck('published net rounding '+period,abs(pn-published)<=D('.001'))
        # Independent exact rational reconstruction of the published gross inputs.
        fractions=[Fraction(r['Value']) for r in rows];fraction_net=fractions[2]-fractions[3]
        ck('independent net '+period,Fraction(pn)==fraction_net)
        result.append(dict(period=period[:4] if frequency=='annual' else period[:4]+'-'+period[4:],frequency=frequency,crude_imports_kbd=ci,crude_exports_kbd=ce,noncrude_imports_kbd=noni,noncrude_exports_kbd=none,petroleum_imports_kbd=pi,petroleum_exports_kbd=pe,crude_net_imports_kbd=cn,noncrude_net_imports_kbd=nonnet,petroleum_net_imports_calculated_kbd=pn,petroleum_net_imports_published_kbd=published,published_net_rounding_difference_kbd=pn-published,unit='thousand barrels per day',source_vintage='October 2026 Monthly Energy Review',import_source=URL['T03.03B'],export_source=URL['T03.03E'],published_net_source=URL['T03.03A']))
    return result
def main():
    raw=source();annual_periods=[f'{y}13' for y in range(1949,2026)];monthly_periods=[f'{y}{m:02d}' for y in range(1977,2026) for m in range(1,13)]
    annual=panel(raw,annual_periods,'annual');monthly=panel(raw,monthly_periods,'monthly')
    write('annual-trade-balances-1949-2025.csv',annual);write('monthly-trade-balances-1977-2025.csv',monthly)
    traces=[]
    for per in annual_periods+monthly_periods:
        for msn in MSNS:
            r=raw[(msn,per)];traces.append(dict(msn=msn,source_period=per,raw_value=r['Value'],description=r['Description'],unit=r['Unit'],source_url=URL['T03.03B' if msn in ['COIMPUS','PAIMPUS'] else 'T03.03E' if msn in ['COEXPUS','PAEXPUS'] else 'T03.03A']))
    write('used-source-observations.csv',traces)
    ratios=[]
    for r in annual:
        ratios.append(dict(year=r['period'],crude_trade_status='net importer' if r['crude_net_imports_kbd']>0 else 'net exporter' if r['crude_net_imports_kbd']<0 else 'balanced',noncrude_trade_status='net importer' if r['noncrude_net_imports_kbd']>0 else 'net exporter' if r['noncrude_net_imports_kbd']<0 else 'balanced',petroleum_trade_status='net importer' if r['petroleum_net_imports_calculated_kbd']>0 else 'net exporter' if r['petroleum_net_imports_calculated_kbd']<0 else 'balanced',total_exports_as_share_of_total_imports_percent=r['petroleum_exports_kbd']/r['petroleum_imports_kbd']*100,crude_share_of_petroleum_imports_percent=r['crude_imports_kbd']/r['petroleum_imports_kbd']*100,crude_share_of_petroleum_exports_percent=r['crude_exports_kbd']/r['petroleum_exports_kbd']*100,noncrude_share_of_petroleum_exports_percent=r['noncrude_exports_kbd']/r['petroleum_exports_kbd']*100,source_url=URL['T03.03B'],additional_source_url=URL['T03.03E']))
    write('annual-status-and-gross-flow-ratios.csv',ratios)
    counts=[]
    for year in range(1977,2026):
        rows=[r for r in monthly if r['period'].startswith(str(year))];ck('twelve months '+str(year),len(rows)==12)
        a=next(r for r in annual if r['period']==str(year));counts.append(dict(year=year,complete_months=12,net_crude_importer_months=sum(r['crude_net_imports_kbd']>0 for r in rows),net_noncrude_exporter_months=sum(r['noncrude_net_imports_kbd']<0 for r in rows),net_total_petroleum_exporter_months=sum(r['petroleum_net_imports_calculated_kbd']<0 for r in rows),annual_total_petroleum_net_imports_kbd=a['petroleum_net_imports_calculated_kbd'],source_url=URL['T03.03A']))
    write('complete-year-monthly-trade-status-counts.csv',counts)
    changes=[]
    for old,new in zip(annual,annual[1:]):
        ci=new['crude_imports_kbd']-old['crude_imports_kbd'];ce=-(new['crude_exports_kbd']-old['crude_exports_kbd']);ni=new['noncrude_imports_kbd']-old['noncrude_imports_kbd'];ne=-(new['noncrude_exports_kbd']-old['noncrude_exports_kbd']);net=new['petroleum_net_imports_calculated_kbd']-old['petroleum_net_imports_calculated_kbd'];ck('annual change bridge '+new['period'],ci+ce+ni+ne==net)
        changes.append(dict(previous_year=old['period'],year=new['period'],change_crude_imports_effect_kbd=ci,change_crude_exports_effect_kbd=ce,change_noncrude_imports_effect_kbd=ni,change_noncrude_exports_effect_kbd=ne,change_total_net_imports_kbd=net,interpretation='Accounting components, not causal effects; export increases reduce net imports',import_source=URL['T03.03B'],export_source=URL['T03.03E']))
    write('annual-net-trade-change-bridge.csv',changes)
    latest=annual[-1];bridge=[]
    for category,prefix in [('Crude oil including lease condensate','crude'),('Non-crude petroleum residual','noncrude'),('Total petroleum','petroleum')]:
        bridge.append(dict(year=2025,category=category,imports_kbd=latest[prefix+'_imports_kbd'],exports_kbd=latest[prefix+'_exports_kbd'],net_imports_kbd=latest[prefix+'_net_imports'+('_calculated' if prefix=='petroleum' else '')+'_kbd'],scope='50 states and District of Columbia; export shipments to and import receipts from U.S. territories included',import_source=URL['T03.03B'],export_source=URL['T03.03E']))
    write('fixed-year-category-bridge-2025.csv',bridge)
    old=next(r for r in annual if r['period']=='2005');improvement=old['petroleum_net_imports_calculated_kbd']-latest['petroleum_net_imports_calculated_kbd'];contributions=[]
    for title,key,sign in [('Lower crude imports','crude_imports_kbd',1),('Higher crude exports','crude_exports_kbd',-1),('Lower non-crude imports','noncrude_imports_kbd',1),('Higher non-crude exports','noncrude_exports_kbd',-1)]:
        change=(latest[key]-old[key])*sign;contributions.append(dict(component=title,start_year=2005,end_year=2025,start_gross_flow_kbd=old[key],end_gross_flow_kbd=latest[key],effect_on_net_imports_kbd=change,share_of_total_net_import_reduction_percent=-change/improvement*100,interpretation='Accounting contribution using fixed2005peak baseline, not causal attribution',import_source=URL['T03.03B'],export_source=URL['T03.03E']))
    ck('2005-2025 bridge',sum(r['effect_on_net_imports_kbd'] for r in contributions)==-improvement);write('net-trade-reduction-contributions-2005-2025.csv',contributions)
    facts={'annual_first_year':1949,'annual_last_year':2025,'annual_years':len(annual),'monthly_first':'1977-01','monthly_last':'2025-12','monthly_observations':len(monthly),'annual_net_crude_importer_years':sum(r['crude_net_imports_kbd']>0 for r in annual),'annual_net_total_exporter_years':sum(r['petroleum_net_imports_calculated_kbd']<0 for r in annual),'first_annual_total_net_exporter':next(r['period'] for r in annual if r['petroleum_net_imports_calculated_kbd']<0),'first_monthly_total_net_exporter_in_fixed_domain':next(r['period'] for r in monthly if r['petroleum_net_imports_calculated_kbd']<0),'monthly_net_total_exporter_count':sum(r['petroleum_net_imports_calculated_kbd']<0 for r in monthly),'monthly_net_crude_importer_count':sum(r['crude_net_imports_kbd']>0 for r in monthly),'latest':latest,'latest_ratios':ratios[-1],'net_import_peak':max(annual,key=lambda r:r['petroleum_net_imports_calculated_kbd']),'net_change_2005_to_2025_kbd':-improvement,'component_bridge':contributions,'maximum_published_rounding_difference_kbd':max(abs(r['published_net_rounding_difference_kbd']) for r in annual+monthly),'source_hashes':{p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'sources').glob('T*.csv')}}
    (DATA/'analysis-summary.json').write_text(json.dumps(facts,indent=2,default=str)+'\n')
    (QA/'analysis-results.json').write_text(json.dumps({'status':'PASS','checks_passed':len(CHECKS),'annual_years':len(annual),'monthly_observations':len(monthly),'source_cells':len(traces),'max_published_net_difference_kbd':str(facts['maximum_published_rounding_difference_kbd'])},indent=2)+'\n')
    print(json.dumps({'status':'PASS','checks':len(CHECKS),'annual_years':len(annual),'months':len(monthly)}))
if __name__=='__main__':main()
