"""AMR public-interest research, pinned November 14, 2026 edition.

Standard-library reproduction: python3 analysis.py
Inputs are resolved relative to this file. No network requests or installation.
The source workbook is read, never recalculated or modified.
"""
from pathlib import Path
from decimal import Decimal, getcontext
from fractions import Fraction
from statistics import median
import csv, io, json, zipfile, hashlib, xml.etree.ElementTree as ET

getcontext().prec=34
ROOT=Path(__file__).resolve().parent
D=Decimal
EIA='https://www.eia.gov/outlooks/steo/report/perspectives/2026/10-winterfuels/wf01.xlsx'
BEA='https://apps.bea.gov/regional/zip/CAINC1.zip'
USGS='https://pubs.usgs.gov/periodicals/mcs2026/mcs2026.pdf'
CHECKS=[]
def check(name,condition):
    CHECKS.append(name)
    assert condition,name
def out(lane,name,rows):
    p=ROOT/lane/'data';p.mkdir(parents=True,exist_ok=True)
    check(lane+'/'+name+' complete',bool(rows) and all(set(r)==set(rows[0]) for r in rows) and all(v is not None and str(v)!='' for r in rows for v in r.values()))
    with (p/name).open('w',newline='') as f:
        w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows)
    return rows
def ratio(a,b):return D(a)/D(b)*100
def xlsx(path):
    ns={'m':'http://schemas.openxmlformats.org/spreadsheetml/2006/main'}
    with zipfile.ZipFile(path) as z:
        strings=[]
        if 'xl/sharedStrings.xml' in z.namelist():
            strings=[''.join(e.itertext()) for e in ET.fromstring(z.read('xl/sharedStrings.xml')).findall('m:si',ns)]
        rel={r.attrib['Id']:r.attrib['Target'] for r in ET.fromstring(z.read('xl/_rels/workbook.xml.rels'))}
        sheets={}
        for s in ET.fromstring(z.read('xl/workbook.xml')).findall('m:sheets/m:sheet',ns):
            target=rel[s.attrib['{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id']]
            target=target.lstrip('/') if target.startswith('/') else 'xl/'+target
            cells={}
            for c in ET.fromstring(z.read(target)).findall('.//m:sheetData/m:row/m:c',ns):
                typ=c.attrib.get('t','n');v=c.find('m:v',ns)
                value=v.text if v is not None else ''
                if typ=='s':value=strings[int(value)]
                if typ=='inlineStr':value=''.join(c.find('m:is',ns).itertext())
                if value!='':cells[c.attrib['r']]=(typ,value)
            sheets[s.attrib['name']]=cells
        return sheets

def heating():
    sheets=xlsx(ROOT/'sources/winter-fuels-2026-11.xlsx');s=sheets['Seasonal WFO'];m=sheets['Monthly WFO']
    check('November workbook edition','November 2026' in s['A2'][1])
    cases=[('Natural gas','Northeast',6,6,'Mcf'),('Natural gas','Midwest',10,14,'Mcf'),('Natural gas','South',14,22,'Mcf'),('Natural gas','West',18,30,'Mcf'),('Natural gas','U.S. average',22,38,'Mcf'),('Heating oil','U.S. average',27,47,'gallons'),('Electricity','Northeast',32,56,'kWh'),('Electricity','Midwest',36,64,'kWh'),('Electricity','South',40,72,'kWh'),('Electricity','West',44,80,'kWh'),('Electricity','U.S. average',48,88,'kWh'),('Propane','Northeast',53,97,'gallons'),('Propane','Midwest',57,105,'gallons'),('Propane','South',61,113,'gallons'),('Propane','U.S. average',65,121,'gallons')]
    sourcecells=[]
    def val(sheet,cell):
        typ,raw=sheets[sheet][cell]
        check('Numeric '+sheet+'!'+cell,typ=='n')
        sourcecells.append(dict(sheet=sheet,cell=cell,cached_value=raw,source_url=EIA))
        return D(raw)
    def v(cell):return val('Seasonal WFO',cell)
    def mv(cell):return val('Monthly WFO',cell)
    panel=[];scenarios=[];monthly=[];enduses=[];decomp=[]
    for fuel,region,r,mr,unit in cases:
        check('Region match '+fuel+region,region.lower().replace(' average','') in s['A'+str(r)][1].lower().replace(' average','') and region.lower().replace(' average','') in m['A'+str(mr)][1].lower().replace(' average',''))
        for i,col in enumerate('BCDEFG'):
            q,p,e=[v(col+str(r+j)) for j in (1,2,3)]
            if fuel=='Electricity':p=p/100
            check('Price quantity identity '+fuel+region+col,abs(q*p-e)<D('0.0000001'))
            panel.append(dict(fuel=fuel,region=region,winter=f'{2021+i}-{str(2022+i)[2:]}',status='forecast' if col=='G' else 'producer estimate in November 2026 vintage',quantity=q,quantity_unit=unit,unit_price_usd=p,expenditure_usd=e,source_cells='Seasonal WFO!'+col+str(r+1)+':'+col+str(r+3),source_url=EIA))
        current={}
        for case,col in [('base','G'),('warmer','I'),('colder','K')]:
            q,p,e=[v(col+str(r+j)) for j in (1,2,3)]
            if fuel=='Electricity':p=p/100
            check('Scenario identity '+fuel+region+case,abs(q*p-e)<D('0.0000001'))
            current[case]=e
            scenarios.append(dict(fuel=fuel,region=region,winter='2026-27',case=case,quantity=q,quantity_unit=unit,unit_price_usd=p,expenditure_usd=e,change_vs_2025_26_percent=(e/v('F'+str(r+3))-1)*100,change_vs_base_percent=(e/v('G'+str(r+3))-1)*100,source_cells='Seasonal WFO!'+col+str(r+1)+':'+col+str(r+3),source_url=EIA))
        q0,p0,e0=[v('F'+str(r+j)) for j in (1,2,3)];q1,p1,e1=[v('G'+str(r+j)) for j in (1,2,3)]
        if fuel=='Electricity':p0=p0/100;p1=p1/100
        price=(p1-p0)*q0;quantity=p0*(q1-q0);interaction=(p1-p0)*(q1-q0)
        check('Decomposition '+fuel+region,abs(e1-e0-price-quantity-interaction)<D('0.0000001'))
        decomp.append(dict(fuel=fuel,region=region,previous_expenditure_usd=e0,base_expenditure_usd=e1,change_usd=e1-e0,price_component_usd=price,quantity_component_usd=quantity,interaction_usd=interaction,price_change_percent=(p1/p0-1)*100,quantity_change_percent=(q1/q0-1)*100,source_url=EIA))
        for year,cols,total in [('2025-26','BCDEF','G'),('2026-27','HIJKL','M')]:
            total_e=mv(total+str(mr+5));heat=mv(total+str(mr+6));nonheat=mv(total+str(mr+7))
            check('End use sum '+fuel+region+year,abs(heat+nonheat-total_e)<D('0.0000001'))
            enduses.append(dict(fuel=fuel,region=region,winter=year,total_expenditure_usd=total_e,heating_expenditure_usd=heat,nonheating_expenditure_usd=nonheat,heating_share_percent=ratio(heat,total_e),nonheating_share_percent=ratio(nonheat,total_e),source_cells='Monthly WFO!'+total+str(mr+5)+':'+total+str(mr+7),source_url=EIA))
            vals=[]
            for col,month in zip(cols,['November','December','January','February','March']):
                quantity,heating_q,nonheating_q,price,exp,heat,nonheat=[mv(col+str(mr+j)) for j in range(1,8)]
                if fuel=='Electricity':price=price/100
                check('Monthly components '+fuel+region+year+month,abs(heat+nonheat-exp)<D('0.0000001') and abs(heating_q+nonheating_q-quantity)<D('0.0000001') and abs(quantity*price-exp)<D('0.0000001'))
                vals.append(exp)
                monthly.append(dict(fuel=fuel,region=region,winter=year,month=month,quantity=quantity,quantity_unit=unit,heating_quantity=heating_q,nonheating_quantity=nonheating_q,unit_price_usd=price,expenditure_usd=exp,heating_expenditure_usd=heat,nonheating_expenditure_usd=nonheat,source_cells='Monthly WFO!'+col+str(mr+1)+':'+col+str(mr+7),source_url=EIA))
            check('Monthly sum '+fuel+region+year,abs(sum(vals)-total_e)<D('0.0000001'))
            check('Monthly seasonal agreement '+fuel+region+year,abs(total_e-v(('F' if year=='2025-26' else 'G')+str(r+3)))<D('0.0000001'))
    households=[];hdd=[]
    for region,r in [('Northeast',70),('Midwest',77),('South',84),('West',91),('United States',98)]:
        for i,col in enumerate('BCDEFG'):
            vals=[v(col+str(r+j)) for j in range(1,7)];total=sum(vals)
            for fuel,n in zip(['Natural gas','Heating oil including kerosene','Propane','Electricity','Wood','Other or none'],vals):
                households.append(dict(region=region,winter=f'{2021+i}-{str(2022+i)[2:]}',main_heating_fuel=fuel,households_thousands=n,share_percent=ratio(n,total),all_fuels_total_thousands=total,source_url=EIA))
    for region,r in [('Northeast',106),('Midwest',107),('South',108),('West',109),('United States',110)]:
        for winter,col in zip(['2021-22','2022-23','2023-24','2024-25','2025-26','2026-27'],'BCDEFG'):
            hdd.append(dict(region=region,winter=winter,case='base',heating_degree_days=v(col+str(r)),source_url=EIA))
        for case,col in [('warmer','I'),('colder','K')]:hdd.append(dict(region=region,winter='2026-27',case=case,heating_degree_days=v(col+str(r)),source_url=EIA))
    for name,rows in [('seasonal-household-expenditures.csv',panel),('weather-scenarios.csv',scenarios),('monthly-household-expenditures.csv',monthly),('heating-and-nonheating.csv',enduses),('price-quantity-decomposition.csv',decomp),('main-heating-fuel-households.csv',households),('heating-degree-days.csv',hdd)]:out('heating',name,rows)
    sourcecells=list({(r['sheet'],r['cell']):r for r in sourcecells}.values());out('heating','used-workbook-cells.csv',sourcecells)
    check('Full heating domains',len(panel)==90 and len(scenarios)==45 and len(monthly)==150 and len(enduses)==30 and len(households)==180)
    return dict(cases=len(cases),seasonal_rows=len(panel),monthly_rows=len(monthly),source_cells=len(sourcecells),national_base=[r for r in scenarios if r['region']=='U.S. average' and r['case']=='base'],national_enduses=[r for r in enduses if r['region']=='U.S. average' and r['winter']=='2026-27'])

def county():
    with zipfile.ZipFile(ROOT/'sources/CAINC1.zip') as z:
        raw=list(csv.DictReader(io.StringIO(z.read('CAINC1__ALL_AREAS_1969_2024.csv').decode('cp1252'))))
    source={}
    for r in raw:
        fips=r['GeoFIPS'].strip(' "')
        if fips.startswith('48') and r['LineCode'] in ['1','2','3']:source[(fips,int(r['LineCode']))]=r
    check('255 complete BEA geographies',len(source)==765)
    cohort=['Midland','Martin','Upton','Loving','Howard','Karnes','Reeves','Reagan','Andrews','Glasscock']
    panel=[];coh=[];state=[];summary=[];latest=[];growth=[]
    for y in range(2010,2025):
        area=[]
        for fips in sorted({k[0] for k in source}):
            income=D(source[(fips,1)][str(y)]);pop=D(source[(fips,2)][str(y)]);pcpi=D(source[(fips,3)][str(y)])
            name=source[(fips,1)]['GeoName'].replace(', TX','').replace('*','').strip()
            check('Positive county input '+fips+str(y),income>0 and pop>0 and pcpi>0)
            calculated=income*1000/pop
            check('County independent quotient '+fips+str(y),abs(calculated-D(Fraction(int(income)*1000,int(pop)).numerator)/D(Fraction(int(income)*1000,int(pop)).denominator))<D('1e-25'))
            check('County reported rounding '+fips+str(y),abs(pcpi-calculated)<=D('.501'))
            area.append(dict(fips=fips,county=name,year=y,personal_income_thousand_usd=income,population=pop,reported_pcpi_usd=pcpi,calculated_income_per_resident_usd=calculated,in_fixed_oil_cohort='yes' if name in cohort else 'no',source_url=BEA))
        st=next(r for r in area if r['fips']=='48000');counties=[r for r in area if r['fips']!='48000']
        check('Texas county count '+str(y),len(counties)==254)
        # Texas state totals may include independent adjustments. Keep the published state denominator.
        state.append(st)
        for r in counties:
            r['texas_rank_competition']=1+sum(x['reported_pcpi_usd']>r['reported_pcpi_usd'] for x in counties)
            r['ratio_to_texas_reported_pcpi']=r['reported_pcpi_usd']/st['reported_pcpi_usd']
        panel.extend(counties);selected=[r for r in counties if r['county'] in cohort];check('Fixed ten '+str(y),len(selected)==10);coh.extend(selected)
        pi=sum(r['personal_income_thousand_usd'] for r in selected);pop=sum(r['population'] for r in selected);weighted=pi*1000/pop
        summary.append(dict(year=y,cohort_count=10,personal_income_thousand_usd=pi,population=pop,population_weighted_income_per_resident_usd=weighted,unweighted_median_county_pcpi_usd=median(r['reported_pcpi_usd'] for r in selected),unweighted_mean_county_pcpi_usd=sum(r['reported_pcpi_usd'] for r in selected)/10,texas_reported_pcpi_usd=st['reported_pcpi_usd'],weighted_ratio_to_texas=weighted/st['reported_pcpi_usd'],texas_population_share_percent=ratio(pop,st['population']),texas_personal_income_share_percent=ratio(pi,st['personal_income_thousand_usd']),counties_above_texas=sum(r['reported_pcpi_usd']>st['reported_pcpi_usd'] for r in selected),source_url=BEA))
    for r in coh:
        if r['year']==2024:latest.append(r)
    for r in latest:
        a=next(x for x in coh if x['county']==r['county'] and x['year']==2010)
        fi=r['personal_income_thousand_usd']/a['personal_income_thousand_usd'];fp=r['population']/a['population'];fpc=r['calculated_income_per_resident_usd']/a['calculated_income_per_resident_usd']
        check('Growth identity '+r['county'],abs(fi/fp-fpc)<D('1e-28'))
        growth.append(dict(county=r['county'],fips=r['fips'],period='2010-2024',income_growth_percent=(fi-1)*100,population_growth_percent=(fp-1)*100,calculated_income_per_resident_growth_percent=(fpc-1)*100,income_factor=fi,population_factor=fp,income_per_resident_factor=fpc,rank_2010=a['texas_rank_competition'],rank_2024=r['texas_rank_competition'],source_url=BEA))
    for name,rows in [('texas-all-counties-2010-2024.csv',panel),('fixed-oil-cohort-2010-2024.csv',coh),('texas-state-2010-2024.csv',state),('cohort-weighted-and-unweighted.csv',summary),('fixed-cohort-2024.csv',sorted(latest,key=lambda r:r['texas_rank_competition'])),('fixed-cohort-growth-2010-2024.csv',growth)]:out('county',name,rows)
    return dict(panel_rows=len(panel),latest=latest,summary_2024=summary[-1],summary_2010=summary[0],growth=growth)

def minerals():
    inp=json.loads((ROOT/'sources/mineral-input.json').read_text());overview=[];origins=[];history=[];world=[]
    for r in inp['materials']:
        shares=r['origins'];check('Origin shares '+r['material'],sum(shares.values())==100)
        named=[(k,v) for k,v in shares.items() if k!='Other'];ranked=sorted(named,key=lambda x:-x[1]);top3=sum(v for k,v in ranked[:3])
        # Other is an aggregate, not a country. This is a lower bound HHI only.
        lower=sum(D(v)**2 for k,v in named);upper=lower+D(shares['Other'])**2
        overview.append(dict(material=r['material'],nir_percent_2025=r['nir'][-1],nir_scope=r['nir_scope'],import_origin_form=r['import_form'],import_origin_period='2021-2024',leading_reported_origin=ranked[0][0],leading_origin_share_percent=ranked[0][1],top_three_named_origins_share_percent=top3,other_origins_share_percent=shares['Other'],us_mine_2025_metric_tons=r['us_mine'],world_mine_2025_metric_tons=r['world_mine'],us_share_of_rounded_world_mine_percent=ratio(r['us_mine'],r['world_mine']),production_basis=r['basis'],pdf_pages=r['pdf_pages'],source_url=USGS))
        for origin,share in shares.items():origins.append(dict(material=r['material'],import_form=r['import_form'],period='2021-2024',origin=origin,share_percent=share,origin_type='reported residual aggregate' if origin=='Other' else 'named country or area',source_url=USGS,pdf_page=r['pdf_pages'].split(';')[0]))
        for year,value in zip(range(2021,2026),r['nir']):
            op='greater than' if str(value).startswith('>') else 'exact as published rounded percent';numeric=str(value).lstrip('>')
            history.append(dict(material=r['material'],year=year,reported_nir=str(value),relation=op,reported_numeric_value=numeric,scope=r['nir_scope'],source_url=USGS,pdf_page=r['pdf_pages'].split(';')[0]))
        for country,qty in r['mine_countries'].items():world.append(dict(material=r['material'],year=2025,country=country,mine_output_metric_tons=qty,share_of_rounded_world_total_percent=ratio(qty,r['world_mine']),basis=r['basis'],status='2025 estimate from MCS2026',source_url=USGS,pdf_page=r['pdf_pages'].split(';')[-1]))
    for name,rows in [('mineral-dependence-crosswalk.csv',overview),('reported-import-origins.csv',origins),('net-import-reliance-2021-2025.csv',history),('selected-world-mine-comparisons.csv',world)]:out('minerals',name,rows)
    stages=[]
    for r in inp['copper_stage_observations']:
        stages.append(dict(**r,year=2025,mine_world_share_percent=ratio(r['mine_metric_tons'],23000000),refinery_world_share_percent=ratio(r['refinery_metric_tons'],29000000),source_pdf_page=77,source_url=USGS))
    out('minerals','copper-mining-refining-stages.csv',stages)
    out('minerals','rare-earth-stage-observations.csv',[dict(**r,year=2025,source_pdf_page=156,source_url=USGS) for r in inp['rare_earth_stage_observations']])
    return dict(materials=5,overview=overview,history_rows=len(history),reported_origin_rows=len(origins))

if __name__=='__main__':
    import argparse
    parser=argparse.ArgumentParser();parser.add_argument('--lane',choices=['all','heating','county','minerals'],default='all');args=parser.parse_args()
    functions={'heating':heating,'county':county,'minerals':minerals};results={k:fn() for k,fn in functions.items() if args.lane in ['all',k]}
    results['checks_passed']=len(CHECKS);results['status']='PASS';results['source_sha256']={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT/'sources').iterdir() if p.name in ['winter-fuels-2026-11.xlsx','CAINC1.zip','mineral-input.json','mcs2026.pdf']}
    (ROOT/'qa').mkdir(exist_ok=True);(ROOT/'qa/analysis-results.json').write_text(json.dumps(results,default=str,indent=2)+'\n')
    print(json.dumps({'status':'PASS','checks':len(CHECKS),'lanes':[k for k in functions if k in results]}))
