PROJECT 024 · PYTHON SOURCE

Unknown category coverage

Study-specific code. Shared modules, dependency versions, and reproduction instructions are included in all project files.

from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))
import numpy as np
import pandas as pd
from scipy.stats import spearmanr
from sklearn.decomposition import PCA
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix, precision_score, recall_score
from portfolio.data import load
from portfolio.constants import *
from portfolio.methods import result, sql, rates, associations, distribution, regression, classification, cluster
from portfolio.engine import execute

META = {'id': 24, 'dataset': 'bank', 'title': 'Unknown category coverage', 'question': 'Which fields contain the most explicit unknown values, and does their response mix differ?', 'method': 'Field-level missing-category coverage and observed outcome comparison.', 'action': 'Improve source capture for fields with material unknown coverage, then reassess model stability.', 'limitations': 'Observational marketing records do not identify campaign uplift. Month and row order are not precise timestamps; repeated-client identifiers are unavailable. Models are research diagnostics, not financial eligibility or automated contact decisions. '}

def analyze():
    df = load('bank').copy()
    rows=[]
    for col in ['job','marital','education','contact','poutcome']:
        mask=df[col].eq('unknown')
        rows.append({'field':col,'unknown_rows':int(mask.sum()),'unknown_share':mask.mean(),'unknown_response':df.loc[mask,'subscribed'].mean(),'known_response':df.loc[~mask,'subscribed'].mean()})
    t=pd.DataFrame(rows)
    out=result(t,'field','unknown_share','Unknown share','A documented unknown category is not the same as an empty CSV cell. Coverage differences may reflect workflow selection.')
    return out

if __name__ == "__main__":
    execute(Path(__file__).parent, META, analyze)