PROJECT 075 · PYTHON SOURCE

Horsepower missingness audit

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 75, 'dataset': 'auto', 'title': 'Horsepower missingness audit', 'question': 'Which model-year and origin groups contain missing horsepower measurements?', 'method': 'Explicit missing indicators summarized over recorded groups.', 'action': 'Review source availability before interpreting imputed horsepower or dropping vehicles.', 'limitations': "Historical city-cycle vehicle observations are not current fleet performance. Vehicle mix and model year are confounded. A '?' horsepower value means missing, not zero. "}

def analyze():
    df = load('auto').copy()
    df['missing_horsepower']=df.horsepower.isna().astype(int)
    t=df.groupby(['model_year','origin']).agg(vehicles=('row_id','size'),missing=('missing_horsepower','sum')).reset_index();t['missing_share']=t.missing/t.vehicles;t=t.sort_values('missing_share',ascending=False);t['group']=t.model_year.astype(str)+' / origin '+t.origin.astype(str)
    out=result(t,'group','missing_share','Missing share','The question-mark marker has been converted to NaN. Zero missingness in a subgroup does not prove representative sampling.')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)